[diffusion] refactor: simplify Kandinsky 6 models, SR sampling and tests (#42768)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
Official out_visual_dim is the total DX head width; n_grid lives in the scheduler.
|
||||
Unsupported architecture options and unknown fields are rejected."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
@@ -16,13 +16,6 @@ ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_param
|
||||
# Bookkeeping keys the loader leaves in the component config.
|
||||
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
|
||||
_SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal")
|
||||
_PIFLOW_FIELDS = (
|
||||
"piflow_nfe",
|
||||
"piflow_num_policy_substeps",
|
||||
"piflow_final_step_size_scale",
|
||||
"piflow_shift",
|
||||
"piflow_eps",
|
||||
)
|
||||
_LQ_NOISE_TYPES = ("ddpm", "linear")
|
||||
|
||||
|
||||
@@ -40,7 +33,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
}
|
||||
)
|
||||
|
||||
# --- DiffusionTransformer3D constructor keys (trained architecture) ---
|
||||
# trained architecture, before inference overrides
|
||||
in_visual_dim: int = 4
|
||||
in_text_dim: int = 3584
|
||||
in_text_dim2: int = 768
|
||||
@@ -61,30 +54,13 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
zero_lq_in_main_path: bool = False
|
||||
use_adapter: bool = False
|
||||
adapter_gamma: float = 0.0
|
||||
# The reference default is True; the SR release is text-free and its config says
|
||||
# so explicitly.
|
||||
use_text: bool = False
|
||||
use_lq_noise_cond: bool = False
|
||||
|
||||
# --- optional legacy / test-only DX head / pi-Flow fields ---
|
||||
# An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the
|
||||
# pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is
|
||||
# ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim).
|
||||
n_grid: int = 1
|
||||
piflow_nfe: int | None = None
|
||||
piflow_num_policy_substeps: int | None = None
|
||||
piflow_final_step_size_scale: float | None = None
|
||||
piflow_shift: float | None = None
|
||||
piflow_eps: float | None = None
|
||||
|
||||
# Post-load attribute overrides (whitelist: instruct_type, visual_cond,
|
||||
# attention_params); the architecture above stays as trained. The official
|
||||
# config writes ``null`` when there are none.
|
||||
# inference-only overrides must not change the trained layer dimensions
|
||||
attribute_overrides: dict[str, Any] | None = field(default_factory=dict)
|
||||
|
||||
# --- SR sampling parameters of the training config ---
|
||||
# As stored in ``transformer/config.json`` (``sr_params``); expanded into the
|
||||
# flat ``sr_*`` fields below in ``__post_init__``.
|
||||
# checkpoint sr_params expands into the flat sr_* fields
|
||||
sr_params: dict[str, Any] | None = None
|
||||
sr_visual_size: list[int] = field(default_factory=lambda: [512])
|
||||
sr_scale_factor: dict[str, list[float]] | None = None
|
||||
@@ -104,77 +80,18 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
if self.attribute_overrides is None:
|
||||
self.attribute_overrides = {}
|
||||
if self.sr_params is not None:
|
||||
self._expand_sr_params(self.sr_params)
|
||||
self._reject_unknown_keys()
|
||||
self._reject_unsupported_flags()
|
||||
self._normalize_and_validate_shapes()
|
||||
self._validate_overrides()
|
||||
self._validate_sampler_fields()
|
||||
self.hidden_size = self.model_dim
|
||||
self.num_attention_heads = self.model_dim // sum(self.axes_dims)
|
||||
self.in_channels = self.in_visual_dim
|
||||
self.out_channels = self.out_visual_dim
|
||||
self.num_channels_latents = self.in_visual_dim
|
||||
|
||||
@property
|
||||
def is_piflow(self) -> bool:
|
||||
return self.piflow_nfe is not None
|
||||
|
||||
@property
|
||||
def head_width(self) -> int:
|
||||
"""Channels the DiT head emits per latent position (all grids of a DX head)."""
|
||||
return self.out_visual_dim * self.n_grid
|
||||
|
||||
@property
|
||||
def base_out_visual_dim(self) -> int:
|
||||
"""Channels per latent grid; legacy n_grid configs store the base output width."""
|
||||
return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim
|
||||
|
||||
@property
|
||||
def trained_wide_input(self) -> bool:
|
||||
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
|
||||
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
|
||||
|
||||
def effective_override(self, name: str) -> Any:
|
||||
"""Post-load value of ``name``: the override if present, else as trained."""
|
||||
if name in self.attribute_overrides:
|
||||
return self.attribute_overrides[name]
|
||||
return getattr(self, name)
|
||||
|
||||
def requested_sparse_attention(self) -> str | None:
|
||||
"""Sparse attention type asked for by ``attention_params``, if any."""
|
||||
params = self.effective_override("attention_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
for value in params.values():
|
||||
attention_type = value.get("type") if isinstance(value, dict) else None
|
||||
if attention_type in _SPARSE_ATTENTION_TYPES:
|
||||
return attention_type
|
||||
return None
|
||||
|
||||
_SR_PARAM_FIELDS = {
|
||||
"visual_size": "sr_visual_size",
|
||||
"scale_factor": "sr_scale_factor",
|
||||
"scheduler_scale": "sr_scheduler_scale",
|
||||
"lq_noise_scale": "sr_lq_noise_scale",
|
||||
"lq_noise_type": "sr_lq_noise_type",
|
||||
"lq_channel_noise_scale": "sr_lq_channel_noise_scale",
|
||||
"cap_noise_timestep": "sr_cap_noise_timestep",
|
||||
"fps": "sr_fps",
|
||||
}
|
||||
|
||||
def _expand_sr_params(self, sr_params: dict[str, Any]) -> None:
|
||||
"""Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields."""
|
||||
unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS))
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted "
|
||||
f"keys: {sorted(self._SR_PARAM_FIELDS)}"
|
||||
)
|
||||
for key, value in sr_params.items():
|
||||
setattr(self, self._SR_PARAM_FIELDS[key], value)
|
||||
|
||||
def _reject_unknown_keys(self) -> None:
|
||||
known = {
|
||||
item.name.removeprefix("sr_")
|
||||
for item in fields(self)
|
||||
if item.name.startswith("sr_") and item.name != "sr_params"
|
||||
}
|
||||
unknown = sorted(set(self.sr_params) - known)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted keys: {sorted(known)}"
|
||||
)
|
||||
for key, value in self.sr_params.items():
|
||||
setattr(self, f"sr_{key}", value)
|
||||
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
@@ -182,23 +99,15 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config."
|
||||
)
|
||||
|
||||
def _reject_unsupported_flags(self) -> None:
|
||||
if self.use_text:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not "
|
||||
"supported; only the text-free SR DiT (use_text=False) is implemented."
|
||||
)
|
||||
if self.use_adapter:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported."
|
||||
)
|
||||
if self.use_lq_modulation:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is "
|
||||
"not supported."
|
||||
)
|
||||
|
||||
def _normalize_and_validate_shapes(self) -> None:
|
||||
for name, enabled in (
|
||||
("use_text", self.use_text),
|
||||
("use_adapter", self.use_adapter),
|
||||
("use_lq_modulation", self.use_lq_modulation),
|
||||
):
|
||||
if enabled:
|
||||
raise NotImplementedError(
|
||||
f"Kandinsky6SR: {name}=True is not supported."
|
||||
)
|
||||
self.patch_size = tuple(self.patch_size)
|
||||
self.axes_dims = tuple(self.axes_dims)
|
||||
if len(self.patch_size) != 3 or self.patch_size[0] != 1:
|
||||
@@ -218,10 +127,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
)
|
||||
if self.model_dim % 2 != 0:
|
||||
raise ValueError(f"model_dim must be even, got {self.model_dim}")
|
||||
if self.n_grid < 1:
|
||||
raise ValueError(f"n_grid must be >= 1, got {self.n_grid}")
|
||||
|
||||
def _validate_overrides(self) -> None:
|
||||
unknown = sorted(
|
||||
set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST)
|
||||
)
|
||||
@@ -239,19 +145,37 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
visual_cond = self.effective_override("visual_cond")
|
||||
if not isinstance(visual_cond, bool):
|
||||
raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}")
|
||||
|
||||
def _validate_sampler_fields(self) -> None:
|
||||
present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None]
|
||||
if present and len(present) != len(_PIFLOW_FIELDS):
|
||||
missing = sorted(set(_PIFLOW_FIELDS) - set(present))
|
||||
raise ValueError(
|
||||
f"pi-Flow config is incomplete: {missing} missing next to {present}"
|
||||
)
|
||||
if self.sr_lq_noise_type not in _LQ_NOISE_TYPES:
|
||||
raise ValueError(
|
||||
f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, "
|
||||
f"got {self.sr_lq_noise_type!r}"
|
||||
)
|
||||
self.hidden_size = self.model_dim
|
||||
self.num_attention_heads = self.model_dim // head_dim
|
||||
self.in_channels = self.in_visual_dim
|
||||
self.out_channels = self.out_visual_dim
|
||||
self.num_channels_latents = self.in_visual_dim
|
||||
|
||||
@property
|
||||
def trained_wide_input(self) -> bool:
|
||||
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
|
||||
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
|
||||
|
||||
def effective_override(self, name: str) -> Any:
|
||||
"""Post-load value of ``name``: the override if present, else as trained."""
|
||||
return self.attribute_overrides.get(name, self.__dict__[name])
|
||||
|
||||
def requested_sparse_attention(self) -> str | None:
|
||||
"""Sparse attention type asked for by ``attention_params``, if any."""
|
||||
params = self.effective_override("attention_params")
|
||||
if isinstance(params, dict):
|
||||
for value in params.values():
|
||||
if (
|
||||
isinstance(value, dict)
|
||||
and value.get("type") in _SPARSE_ATTENTION_TYPES
|
||||
):
|
||||
return value["type"]
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -157,21 +157,15 @@ class Kandinsky6TI2VAPipelineConfig(PipelineConfig):
|
||||
audio_downsample_factor: int = 1024
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.text_encoder_configs) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoders "
|
||||
f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}."
|
||||
)
|
||||
if len(self.text_encoder_precisions) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoder "
|
||||
f"precisions, but got {len(self.text_encoder_precisions)}."
|
||||
)
|
||||
if len(self.text_encoder_extra_args) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoder extra "
|
||||
f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}."
|
||||
)
|
||||
for name, values in (
|
||||
("text encoders (Reason1 and CLIP)", self.text_encoder_configs),
|
||||
("text encoder precisions", self.text_encoder_precisions),
|
||||
("text encoder extra tokenizer arg dicts", self.text_encoder_extra_args),
|
||||
):
|
||||
if len(values) != 2:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 pipeline requires exactly 2 {name}, but got {len(values)}."
|
||||
)
|
||||
|
||||
# keep the video VAE encoder available for optional image conditioning
|
||||
self.vae_config.load_encoder = True
|
||||
|
||||
@@ -10,11 +10,14 @@ NABLA attention is not supported.
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
from torch.utils.checkpoint import checkpoint
|
||||
|
||||
from sglang.kernels.ops.diffusion import apply_matrix_rope, residual_gate_fp32
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
@@ -69,6 +72,28 @@ def _is_kandinsky6_transformer_block(name: str, module: object) -> bool:
|
||||
return is_module_list_entry_in(name, _KANDINSKY6_BLOCK_CONTAINERS)
|
||||
|
||||
|
||||
def _validate_parallelism(num_heads: int, **tp_dimensions: int) -> None:
|
||||
"""TP splits heads/FFNs; Ulysses splits video heads remaining after TP."""
|
||||
tp_size = get_tp_world_size()
|
||||
ulysses_size, _ = get_ulysses_ctx()
|
||||
ring_size, _ = get_ring_ctx()
|
||||
for name, size in (("TP", tp_size), ("Ulysses", ulysses_size), ("ring", ring_size)):
|
||||
if size <= 0:
|
||||
raise ValueError(f"Kandinsky6 {name} size must be positive.")
|
||||
for name, value in dict(num_attention_heads=num_heads, **tp_dimensions).items():
|
||||
if value % tp_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}."
|
||||
)
|
||||
# ring rotates complete K/V shards, so it imposes no head divisibility constraint
|
||||
local_heads = num_heads // tp_size
|
||||
if local_heads % ulysses_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 TP-local video attention heads {local_heads} must be "
|
||||
f"divisible by Ulysses size {ulysses_size} (total heads={num_heads}, TP={tp_size})."
|
||||
)
|
||||
|
||||
|
||||
def _build_rotary_freqs(dim: int, max_period: float) -> torch.Tensor:
|
||||
return torch.exp(
|
||||
-math.log(max_period)
|
||||
@@ -146,20 +171,13 @@ class Kandinsky6VisualEmbeddings(nn.Module):
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, duration, height, width, dim = x.shape
|
||||
x = (
|
||||
x.view(
|
||||
batch_size,
|
||||
duration // self.patch_size[0],
|
||||
self.patch_size[0],
|
||||
height // self.patch_size[1],
|
||||
self.patch_size[1],
|
||||
width // self.patch_size[2],
|
||||
self.patch_size[2],
|
||||
dim,
|
||||
)
|
||||
.permute(0, 1, 3, 5, 2, 4, 6, 7)
|
||||
.flatten(4, 7)
|
||||
pt, ph, pw = self.patch_size
|
||||
x = rearrange(
|
||||
x,
|
||||
"b (t pt) (h ph) (w pw) c -> b t h w (pt ph pw c)",
|
||||
pt=pt,
|
||||
ph=ph,
|
||||
pw=pw,
|
||||
)
|
||||
x, _ = self.in_layer(x)
|
||||
return x
|
||||
@@ -321,30 +339,22 @@ class Kandinsky6Attention(nn.Module):
|
||||
tp_size = get_tp_world_size()
|
||||
self.local_num_heads = divide(self.num_heads, tp_size)
|
||||
|
||||
self.to_query = ColumnParallelLinear(
|
||||
num_channels,
|
||||
num_channels,
|
||||
bias=True,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_query", prefix),
|
||||
)
|
||||
self.to_key = ColumnParallelLinear(
|
||||
kv_dim,
|
||||
num_channels,
|
||||
bias=True,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_key", prefix),
|
||||
)
|
||||
self.to_value = ColumnParallelLinear(
|
||||
kv_dim,
|
||||
num_channels,
|
||||
bias=True,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("to_value", prefix),
|
||||
)
|
||||
for name, width in (
|
||||
("to_query", num_channels),
|
||||
("to_key", kv_dim),
|
||||
("to_value", kv_dim),
|
||||
):
|
||||
self.add_module(
|
||||
name,
|
||||
ColumnParallelLinear(
|
||||
width,
|
||||
num_channels,
|
||||
bias=True,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix(name, prefix),
|
||||
),
|
||||
)
|
||||
self.query_norm = Kandinsky6QKNorm(head_dim)
|
||||
self.key_norm = Kandinsky6QKNorm(head_dim)
|
||||
self.out_layer = RowParallelLinear(
|
||||
@@ -414,48 +424,20 @@ class Kandinsky6Attention(nn.Module):
|
||||
key = gather_seq(key, context_seq_len)
|
||||
value = gather_seq(value, context_seq_len)
|
||||
|
||||
try:
|
||||
hidden_states = self.attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask_meta=attn_mask_meta,
|
||||
skip_sequence_parallel_override=skip_sequence_parallel,
|
||||
)
|
||||
except AssertionError as exc:
|
||||
# standalone single-rank parity uses SDPA without a pipeline forward context
|
||||
if "Forward context is not set" not in str(exc):
|
||||
raise
|
||||
|
||||
query_shape = query.shape[:-2]
|
||||
key_shape = key.shape[:-2]
|
||||
query = query.reshape(
|
||||
query_shape[0], -1, self.local_num_heads, query.shape[-1]
|
||||
).transpose(1, 2)
|
||||
key = key.reshape(
|
||||
key_shape[0], -1, self.local_num_heads, key.shape[-1]
|
||||
).transpose(1, 2)
|
||||
value = value.reshape(
|
||||
key_shape[0], -1, self.local_num_heads, value.shape[-1]
|
||||
).transpose(1, 2)
|
||||
hidden_states = F.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=None,
|
||||
is_causal=False,
|
||||
)
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(
|
||||
*query_shape, self.local_num_heads, -1
|
||||
)
|
||||
|
||||
hidden_states = self.attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask_meta=attn_mask_meta,
|
||||
skip_sequence_parallel_override=skip_sequence_parallel,
|
||||
)
|
||||
hidden_states = hidden_states.flatten(-2, -1)
|
||||
hidden_states, _ = self.out_layer(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _Kandinsky6MLP(nn.Module):
|
||||
"""Bias-free TP MLP with checkpoint-mapped fc_in/fc_out names.
|
||||
class Kandinsky6FeedForward(nn.Module):
|
||||
"""Bias-free TP MLP with checkpoint-mapped mlp.fc_in/fc_out names.
|
||||
|
||||
The shared MLP currently hardcodes bias=True, so it cannot be used here."""
|
||||
|
||||
@@ -467,7 +449,9 @@ class _Kandinsky6MLP(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.fc_in = ColumnParallelLinear(
|
||||
prefix = add_prefix("mlp", prefix)
|
||||
self.mlp = nn.Module()
|
||||
self.mlp.fc_in = ColumnParallelLinear(
|
||||
dim,
|
||||
ff_dim,
|
||||
bias=False,
|
||||
@@ -475,8 +459,8 @@ class _Kandinsky6MLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc_in", prefix),
|
||||
)
|
||||
self.act = get_act_fn("gelu")
|
||||
self.fc_out = RowParallelLinear(
|
||||
self.mlp.act = get_act_fn("gelu")
|
||||
self.mlp.fc_out = RowParallelLinear(
|
||||
ff_dim,
|
||||
dim,
|
||||
bias=False,
|
||||
@@ -486,29 +470,12 @@ class _Kandinsky6MLP(nn.Module):
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, _ = self.fc_in(x)
|
||||
x = self.act(x)
|
||||
x, _ = self.fc_out(x)
|
||||
x, _ = self.mlp.fc_in(x)
|
||||
x = self.mlp.act(x)
|
||||
x, _ = self.mlp.fc_out(x)
|
||||
return x
|
||||
|
||||
|
||||
class Kandinsky6FeedForward(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
ff_dim: int,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.mlp = _Kandinsky6MLP(
|
||||
dim, ff_dim, prefix=add_prefix("mlp", prefix), quant_config=quant_config
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.mlp(x)
|
||||
|
||||
|
||||
def _norm_scale_shift(
|
||||
norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
@@ -535,7 +502,11 @@ class Kandinsky6OutLayer(nn.Module):
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
|
||||
self,
|
||||
visual_embed: torch.Tensor,
|
||||
time_embed: torch.Tensor,
|
||||
*,
|
||||
compute_dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
shift, scale = torch.chunk(
|
||||
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
|
||||
@@ -543,28 +514,18 @@ class Kandinsky6OutLayer(nn.Module):
|
||||
visual_embed = (
|
||||
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
|
||||
+ shift.float()[:, None, None]
|
||||
).type_as(visual_embed)
|
||||
).to(dtype=compute_dtype or visual_embed.dtype)
|
||||
|
||||
x, _ = self.out_layer(visual_embed)
|
||||
|
||||
batch_size, duration, height, width, _ = x.shape
|
||||
x = (
|
||||
x.view(
|
||||
batch_size,
|
||||
duration,
|
||||
height,
|
||||
width,
|
||||
-1,
|
||||
self.patch_size[0],
|
||||
self.patch_size[1],
|
||||
self.patch_size[2],
|
||||
)
|
||||
.permute(0, 1, 5, 2, 6, 3, 7, 4)
|
||||
.flatten(1, 2)
|
||||
.flatten(2, 3)
|
||||
.flatten(3, 4)
|
||||
pt, ph, pw = self.patch_size
|
||||
return rearrange(
|
||||
x,
|
||||
"b t h w (c pt ph pw) -> b (t pt) (h ph) (w pw) c",
|
||||
pt=pt,
|
||||
ph=ph,
|
||||
pw=pw,
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
class Kandinsky6OutLayerAudio(nn.Module):
|
||||
@@ -591,8 +552,12 @@ class Kandinsky6OutLayerAudio(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
class Kandinsky6TransformerEncoderBlock(nn.Module):
|
||||
"""Text-only self-attention + feed-forward block (video/audio text towers)."""
|
||||
class Kandinsky6TransformerBlock(nn.Module):
|
||||
"""Shared parameter layout for text, video and SR transformer blocks."""
|
||||
|
||||
modulation_name = "visual_modulation"
|
||||
with_cross_attention = False
|
||||
skip_sequence_parallel = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -605,24 +570,38 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.text_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
|
||||
|
||||
self.self_attention_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
self.add_module(
|
||||
self.modulation_name,
|
||||
Kandinsky6Modulation(
|
||||
time_dim, model_dim, 9 if self.with_cross_attention else 6
|
||||
),
|
||||
)
|
||||
self.self_attention = Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=add_prefix("self_attention", prefix),
|
||||
quant_config=quant_config,
|
||||
# text is replicated across SP ranks, so skip Ulysses even for self-attention
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
|
||||
self.feed_forward_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
attention_names = (
|
||||
("self_attention", "cross_attention")
|
||||
if self.with_cross_attention
|
||||
else ("self_attention",)
|
||||
)
|
||||
for name in (*attention_names, "feed_forward"):
|
||||
self.add_module(
|
||||
f"{name}_norm",
|
||||
LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
),
|
||||
)
|
||||
for name in attention_names:
|
||||
self.add_module(
|
||||
name,
|
||||
Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends,
|
||||
prefix=add_prefix(name, prefix),
|
||||
quant_config=quant_config,
|
||||
is_cross_attention=name == "cross_attention",
|
||||
skip_sequence_parallel=self.skip_sequence_parallel
|
||||
or name == "cross_attention",
|
||||
),
|
||||
)
|
||||
self.feed_forward = Kandinsky6FeedForward(
|
||||
model_dim,
|
||||
ff_dim,
|
||||
@@ -630,6 +609,13 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6TransformerEncoderBlock(Kandinsky6TransformerBlock):
|
||||
"""Replicated text self-attention and FFN."""
|
||||
|
||||
modulation_name = "text_modulation"
|
||||
skip_sequence_parallel = True
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, time_embed: torch.Tensor, rope: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
@@ -649,59 +635,10 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
class Kandinsky6TransformerDecoderBlock(nn.Module):
|
||||
"""Self-attention, text cross-attention and FFN.
|
||||
class Kandinsky6TransformerDecoderBlock(Kandinsky6TransformerBlock):
|
||||
"""Video self-attention, text cross-attention and FFN."""
|
||||
|
||||
The joint decoder invokes these sublayers separately to insert cross-modal attention."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dim: int,
|
||||
time_dim: int,
|
||||
ff_dim: int,
|
||||
head_dim: int,
|
||||
supported_attention_backends: set[AttentionBackendEnum] | None = None,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 9)
|
||||
|
||||
self.self_attention_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.self_attention = Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=add_prefix("self_attention", prefix),
|
||||
quant_config=quant_config,
|
||||
# Video self-attention over the (long) visual token sequence --
|
||||
# the one role this port's sequence parallelism actually shards.
|
||||
)
|
||||
|
||||
self.cross_attention_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.cross_attention = Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=add_prefix("cross_attention", prefix),
|
||||
quant_config=quant_config,
|
||||
# video queries attend to replicated text K/V
|
||||
is_cross_attention=True,
|
||||
)
|
||||
|
||||
self.feed_forward_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.feed_forward = Kandinsky6FeedForward(
|
||||
model_dim,
|
||||
ff_dim,
|
||||
prefix=add_prefix("feed_forward", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
with_cross_attention = True
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -749,12 +686,6 @@ class Kandinsky6TransformerDecoderBlock(nn.Module):
|
||||
return visual_embed
|
||||
|
||||
|
||||
def _apply_gate_sum(
|
||||
x: torch.Tensor, out: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return residual_gate_fp32(x, out, gate)
|
||||
|
||||
|
||||
def _apply_scale_shift(
|
||||
norm: nn.LayerNorm, x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
@@ -856,7 +787,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
self.videoT.visual_modulation(t_v).unsqueeze(dim=1), 3, dim=-1
|
||||
)
|
||||
shift, scale, gate = torch.chunk(sa_p, 3, dim=-1)
|
||||
vis = _apply_gate_sum(
|
||||
vis = residual_gate_fp32(
|
||||
vis,
|
||||
self.videoT.self_attention(
|
||||
_norm_scale_shift(self.videoT.self_attention_norm, vis, shift, scale),
|
||||
@@ -877,7 +808,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
self.audioT.visual_modulation(t_a).unsqueeze(dim=1), 3, dim=-1
|
||||
)
|
||||
shift, scale, gate = torch.chunk(sa_p, 3, dim=-1)
|
||||
aud = _apply_gate_sum(
|
||||
aud = residual_gate_fp32(
|
||||
aud,
|
||||
self.audioT.self_attention(
|
||||
_norm_scale_shift(self.audioT.self_attention_norm, aud, shift, scale),
|
||||
@@ -893,7 +824,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
aud_out_t = self.audioT.cross_attention(
|
||||
aud_pre_ca, encoder_hidden_states=text_a
|
||||
)
|
||||
aud = _apply_gate_sum(aud, aud_out_t, gate_a)
|
||||
aud = residual_gate_fp32(aud, aud_out_t, gate_a)
|
||||
|
||||
t_va_mod = t_a if not self.fix_modulation else t_v
|
||||
t_av_mod = t_v if not self.fix_modulation else t_a
|
||||
@@ -910,7 +841,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
va_shift, va_scale, va_gate = torch.chunk(va_params, 3, dim=-1)
|
||||
av_shift, av_scale, av_gate = torch.chunk(av_params, 3, dim=-1)
|
||||
|
||||
vis = _apply_gate_sum(vis, vis_out_t, gate_v)
|
||||
vis = residual_gate_fp32(vis, vis_out_t, gate_v)
|
||||
vis_for_va = _apply_scale_shift(self.va_normalization, vis, va_scale, va_shift)
|
||||
aud_for_av = _apply_scale_shift(self.av_normalization, aud, av_scale, av_shift)
|
||||
rq_v = vis_rope if self.ca_rope else None
|
||||
@@ -928,19 +859,19 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
rotary_emb_kv=rq_v,
|
||||
context_seq_len=video_seq_len,
|
||||
)
|
||||
vis = _apply_gate_sum(
|
||||
vis = residual_gate_fp32(
|
||||
vis,
|
||||
vis_from_aud,
|
||||
(va_gate if not self.cross_gates else av_gate) * va_gate_scale,
|
||||
)
|
||||
aud = _apply_gate_sum(
|
||||
aud = residual_gate_fp32(
|
||||
aud,
|
||||
aud_from_vis,
|
||||
(av_gate if not self.cross_gates else va_gate) * av_gate_scale,
|
||||
)
|
||||
|
||||
shift, scale, gate = torch.chunk(ff_p, 3, dim=-1)
|
||||
vis = _apply_gate_sum(
|
||||
vis = residual_gate_fp32(
|
||||
vis,
|
||||
self.videoT.feed_forward(
|
||||
_norm_scale_shift(self.videoT.feed_forward_norm, vis, shift, scale)
|
||||
@@ -948,7 +879,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
|
||||
gate,
|
||||
)
|
||||
shift, scale, gate = torch.chunk(ff_p_a, 3, dim=-1)
|
||||
aud = _apply_gate_sum(
|
||||
aud = residual_gate_fp32(
|
||||
aud,
|
||||
self.audioT.feed_forward(
|
||||
_norm_scale_shift(self.audioT.feed_forward_norm, aud, shift, scale)
|
||||
@@ -977,42 +908,6 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_tp_config(
|
||||
*, arch: Kandinsky6ArchConfig, tp_size: int, num_heads: int, num_heads_a: int
|
||||
) -> None:
|
||||
if tp_size <= 0:
|
||||
raise ValueError("Kandinsky6 TP size must be positive.")
|
||||
for name, value in (
|
||||
("num_attention_heads (video)", num_heads),
|
||||
("num_attention_heads_a (audio)", num_heads_a),
|
||||
("ff_dim", arch.ff_dim),
|
||||
("ff_dim_a", arch.ff_dim_a),
|
||||
):
|
||||
if value % tp_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_sequence_parallel_config(
|
||||
*, tp_size: int, num_heads: int, ulysses_size: int, ring_size: int
|
||||
) -> None:
|
||||
if ulysses_size <= 0:
|
||||
raise ValueError("Kandinsky6 Ulysses size must be positive.")
|
||||
if ring_size <= 0:
|
||||
raise ValueError("Kandinsky6 ring size must be positive.")
|
||||
if ulysses_size == 1 and ring_size == 1:
|
||||
return
|
||||
# only Ulysses splits heads; ring rotates complete K/V shards
|
||||
local_heads = num_heads // tp_size
|
||||
if local_heads % ulysses_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 TP-local video attention heads {local_heads} must be "
|
||||
f"divisible by Ulysses size {ulysses_size} (total video heads="
|
||||
f"{num_heads}, TP={tp_size})."
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Kandinsky6VideoAudioConfig,
|
||||
@@ -1034,20 +929,11 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
head_dim = sum(arch.axes_dims)
|
||||
head_dim_a = sum(arch.axes_dims_a)
|
||||
|
||||
tp_size = get_tp_world_size()
|
||||
ulysses_size, _ = get_ulysses_ctx()
|
||||
ring_size, _ = get_ring_ctx()
|
||||
self._validate_tp_config(
|
||||
arch=arch,
|
||||
tp_size=tp_size,
|
||||
num_heads=arch.model_dim // head_dim,
|
||||
num_heads_a=arch.model_dim_a // head_dim_a,
|
||||
)
|
||||
self._validate_sequence_parallel_config(
|
||||
tp_size=tp_size,
|
||||
num_heads=arch.model_dim // head_dim,
|
||||
ulysses_size=ulysses_size,
|
||||
ring_size=ring_size,
|
||||
_validate_parallelism(
|
||||
arch.model_dim // head_dim,
|
||||
num_attention_heads_a=arch.model_dim_a // head_dim_a,
|
||||
ff_dim=arch.ff_dim,
|
||||
ff_dim_a=arch.ff_dim_a,
|
||||
)
|
||||
self.in_visual_dim = arch.in_visual_dim
|
||||
self.in_audio_dim = arch.in_audio_dim
|
||||
@@ -1079,46 +965,10 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
)
|
||||
|
||||
if not self.is_multimodal:
|
||||
self.time_embeddings = Kandinsky6TimeEmbeddings(
|
||||
arch.model_dim, arch.time_dim
|
||||
)
|
||||
self.text_embeddings = Kandinsky6TextEmbeddings(
|
||||
arch.in_text_dim, arch.model_dim
|
||||
)
|
||||
self.pooled_text_embeddings = Kandinsky6TextEmbeddings(
|
||||
arch.in_text_dim2, arch.time_dim
|
||||
)
|
||||
self.text_rope_embeddings = Kandinsky6RoPE1D(head_dim)
|
||||
self.text_transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
Kandinsky6TransformerEncoderBlock(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.ff_dim,
|
||||
head_dim,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(f"text_transformer_blocks.{i}", self.prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_text_blocks)
|
||||
]
|
||||
)
|
||||
self.visual_transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
Kandinsky6TransformerDecoderBlock(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.ff_dim,
|
||||
head_dim,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(
|
||||
f"visual_transformer_blocks.{i}", self.prefix
|
||||
),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_visual_blocks)
|
||||
]
|
||||
self._build_text_tower(
|
||||
"", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim
|
||||
)
|
||||
block_cls = Kandinsky6TransformerDecoderBlock
|
||||
else:
|
||||
self.audio_embeddings = Kandinsky6TextEmbeddings(
|
||||
arch.in_audio_dim, arch.model_dim_a
|
||||
@@ -1132,81 +982,35 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
arch.out_audio_dim or arch.in_audio_dim,
|
||||
)
|
||||
|
||||
for (
|
||||
tower_prefix,
|
||||
tower_model_dim,
|
||||
tower_time_dim,
|
||||
tower_head_dim,
|
||||
tower_ff_dim,
|
||||
) in (
|
||||
("video", arch.model_dim, arch.time_dim, head_dim, arch.ff_dim),
|
||||
("audio", arch.model_dim_a, arch.time_dim_a, head_dim_a, arch.ff_dim_a),
|
||||
):
|
||||
setattr(
|
||||
self,
|
||||
f"{tower_prefix}_time_embeddings",
|
||||
Kandinsky6TimeEmbeddings(tower_model_dim, tower_time_dim),
|
||||
)
|
||||
setattr(
|
||||
self,
|
||||
f"{tower_prefix}_text_embeddings",
|
||||
Kandinsky6TextEmbeddings(arch.in_text_dim, tower_model_dim),
|
||||
)
|
||||
setattr(
|
||||
self,
|
||||
f"{tower_prefix}_pooled_text_embeddings",
|
||||
Kandinsky6TextEmbeddings(arch.in_text_dim2, tower_time_dim),
|
||||
)
|
||||
setattr(
|
||||
self,
|
||||
f"{tower_prefix}_text_rope_embeddings",
|
||||
Kandinsky6RoPE1D(tower_head_dim),
|
||||
)
|
||||
setattr(
|
||||
self,
|
||||
f"{tower_prefix}_text_transformer_blocks",
|
||||
nn.ModuleList(
|
||||
[
|
||||
Kandinsky6TransformerEncoderBlock(
|
||||
tower_model_dim,
|
||||
tower_time_dim,
|
||||
tower_ff_dim,
|
||||
tower_head_dim,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(
|
||||
f"{tower_prefix}_text_transformer_blocks.{i}",
|
||||
self.prefix,
|
||||
),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_text_blocks)
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
self.visual_transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
Kandinsky6FusedTransformerDecoderBlock(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.ff_dim,
|
||||
head_dim,
|
||||
arch.model_dim_a,
|
||||
arch.time_dim_a,
|
||||
arch.ff_dim_a,
|
||||
head_dim_a,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(
|
||||
f"visual_transformer_blocks.{i}", self.prefix
|
||||
),
|
||||
ca_rope=arch.ca_rope,
|
||||
cross_gates=arch.cross_gates,
|
||||
fix_modulation=arch.fix_modulation,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_visual_blocks)
|
||||
]
|
||||
self._build_text_tower(
|
||||
"video_", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim
|
||||
)
|
||||
self._build_text_tower(
|
||||
"audio_", arch.model_dim_a, arch.time_dim_a, arch.ff_dim_a, head_dim_a
|
||||
)
|
||||
|
||||
block_cls = partial(
|
||||
Kandinsky6FusedTransformerDecoderBlock,
|
||||
model_dim_a=arch.model_dim_a,
|
||||
time_dim_a=arch.time_dim_a,
|
||||
ff_dim_a=arch.ff_dim_a,
|
||||
head_dim_a=head_dim_a,
|
||||
ca_rope=arch.ca_rope,
|
||||
cross_gates=arch.cross_gates,
|
||||
fix_modulation=arch.fix_modulation,
|
||||
)
|
||||
self.visual_transformer_blocks = nn.ModuleList(
|
||||
block_cls(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.ff_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=self._supported_attention_backends,
|
||||
prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_visual_blocks)
|
||||
)
|
||||
|
||||
self.gradient_checkpointing = False
|
||||
self.hidden_size = arch.hidden_size
|
||||
@@ -1219,6 +1023,38 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
) + ["visual_transformer_blocks"]
|
||||
self.__post_init__()
|
||||
|
||||
def _build_text_tower(self, prefix, model_dim, time_dim, ff_dim, head_dim):
|
||||
arch = self.config
|
||||
self.add_module(
|
||||
f"{prefix}time_embeddings", Kandinsky6TimeEmbeddings(model_dim, time_dim)
|
||||
)
|
||||
self.add_module(
|
||||
f"{prefix}text_embeddings",
|
||||
Kandinsky6TextEmbeddings(arch.in_text_dim, model_dim),
|
||||
)
|
||||
self.add_module(
|
||||
f"{prefix}pooled_text_embeddings",
|
||||
Kandinsky6TextEmbeddings(arch.in_text_dim2, time_dim),
|
||||
)
|
||||
self.add_module(f"{prefix}text_rope_embeddings", Kandinsky6RoPE1D(head_dim))
|
||||
self.add_module(
|
||||
f"{prefix}text_transformer_blocks",
|
||||
nn.ModuleList(
|
||||
Kandinsky6TransformerEncoderBlock(
|
||||
model_dim,
|
||||
time_dim,
|
||||
ff_dim,
|
||||
head_dim,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(
|
||||
f"{prefix}text_transformer_blocks.{i}", self.prefix
|
||||
),
|
||||
quant_config=self.quant_config,
|
||||
)
|
||||
for i in range(arch.num_text_blocks)
|
||||
),
|
||||
)
|
||||
|
||||
def _encode_text(
|
||||
self,
|
||||
prefix: str | None,
|
||||
@@ -1237,11 +1073,8 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
text_rope = rope_embeddings(text_rope_pos).unsqueeze(dim=0)
|
||||
for block in blocks:
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
te = torch.utils.checkpoint.checkpoint(
|
||||
block, te, tm, text_rope, use_reentrant=False
|
||||
)
|
||||
else:
|
||||
te = block(te, tm, text_rope)
|
||||
block = partial(checkpoint, block, use_reentrant=False)
|
||||
te = block(te, tm, text_rope)
|
||||
return te, tm
|
||||
|
||||
def forward(
|
||||
@@ -1332,51 +1165,27 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
audio_rope = self.audio_rope_embeddings(audio_rope_pos).unsqueeze(dim=0)
|
||||
for block in self.visual_transformer_blocks:
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
visual_embed, audio_embed = torch.utils.checkpoint.checkpoint(
|
||||
block,
|
||||
visual_embed,
|
||||
audio_embed,
|
||||
video_te,
|
||||
audio_te,
|
||||
(video_tm, audio_tm),
|
||||
visual_rope,
|
||||
audio_rope,
|
||||
va_gate_scale,
|
||||
av_gate_scale,
|
||||
video_seq_len=shard.orig_len,
|
||||
video_attn_meta=attn_meta,
|
||||
use_reentrant=False,
|
||||
)
|
||||
else:
|
||||
visual_embed, audio_embed = block(
|
||||
visual_embed,
|
||||
audio_embed,
|
||||
video_te,
|
||||
audio_te,
|
||||
(video_tm, audio_tm),
|
||||
visual_rope,
|
||||
audio_rope,
|
||||
va_gate_scale,
|
||||
av_gate_scale,
|
||||
video_seq_len=shard.orig_len,
|
||||
video_attn_meta=attn_meta,
|
||||
)
|
||||
block = partial(checkpoint, block, use_reentrant=False)
|
||||
visual_embed, audio_embed = block(
|
||||
visual_embed,
|
||||
audio_embed,
|
||||
video_te,
|
||||
audio_te,
|
||||
(video_tm, audio_tm),
|
||||
visual_rope,
|
||||
audio_rope,
|
||||
va_gate_scale,
|
||||
av_gate_scale,
|
||||
video_seq_len=shard.orig_len,
|
||||
video_attn_meta=attn_meta,
|
||||
)
|
||||
else:
|
||||
for block in self.visual_transformer_blocks:
|
||||
if torch.is_grad_enabled() and self.gradient_checkpointing:
|
||||
visual_embed = torch.utils.checkpoint.checkpoint(
|
||||
block,
|
||||
visual_embed,
|
||||
video_te,
|
||||
video_tm,
|
||||
visual_rope,
|
||||
attn_mask_meta=attn_meta,
|
||||
use_reentrant=False,
|
||||
)
|
||||
else:
|
||||
visual_embed = block(
|
||||
visual_embed, video_te, video_tm, visual_rope, attn_meta
|
||||
)
|
||||
block = partial(checkpoint, block, use_reentrant=False)
|
||||
visual_embed = block(
|
||||
visual_embed, video_te, video_tm, visual_rope, attn_meta
|
||||
)
|
||||
|
||||
visual_embed = gather_seq(visual_embed, shard.orig_len).reshape(
|
||||
*visual_shape, -1
|
||||
|
||||
@@ -12,16 +12,11 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
ATTRIBUTE_OVERRIDE_WHITELIST,
|
||||
WIDE_INPUT_INSTRUCT_TYPES,
|
||||
Kandinsky6SRArchConfig,
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
|
||||
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_ring_ctx,
|
||||
get_ulysses_ctx,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
|
||||
gather_seq,
|
||||
shard_like,
|
||||
@@ -38,14 +33,13 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6Attention,
|
||||
Kandinsky6FeedForward,
|
||||
Kandinsky6Modulation,
|
||||
Kandinsky6OutLayer,
|
||||
Kandinsky6RoPE3D,
|
||||
Kandinsky6TimeEmbeddings,
|
||||
Kandinsky6TransformerBlock,
|
||||
Kandinsky6VisualEmbeddings,
|
||||
_build_rotary_freqs,
|
||||
_validate_parallelism,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
@@ -80,40 +74,8 @@ class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D):
|
||||
return self
|
||||
|
||||
|
||||
class Kandinsky6SRDecoderBlock(nn.Module):
|
||||
"""Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dim: int,
|
||||
time_dim: int,
|
||||
ff_dim: int,
|
||||
head_dim: int,
|
||||
supported_attention_backends: set[AttentionBackendEnum] | None = None,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
|
||||
self.self_attention_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.self_attention = Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=add_prefix("self_attention", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.feed_forward_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.feed_forward = Kandinsky6FeedForward(
|
||||
model_dim,
|
||||
ff_dim,
|
||||
prefix=add_prefix("feed_forward", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
class Kandinsky6SRDecoderBlock(Kandinsky6TransformerBlock):
|
||||
"""Modulated self-attention and FFN with fp32 residuals."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -139,32 +101,6 @@ class Kandinsky6SRDecoderBlock(nn.Module):
|
||||
return visual_embed + gate.float() * out.float()
|
||||
|
||||
|
||||
class Kandinsky6SROutLayer(Kandinsky6OutLayer):
|
||||
"""K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype)."""
|
||||
|
||||
def forward(
|
||||
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
shift, scale = torch.chunk(
|
||||
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
|
||||
)
|
||||
x = (
|
||||
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
|
||||
+ shift.float()[:, None, None]
|
||||
)
|
||||
x, _ = self.out_layer(x.to(self.out_layer.weight.dtype))
|
||||
|
||||
batch_size, duration, height, width, _ = x.shape
|
||||
pt, ph, pw = self.patch_size
|
||||
return (
|
||||
x.view(batch_size, duration, height, width, -1, pt, ph, pw)
|
||||
.permute(0, 1, 5, 2, 6, 3, 7, 4)
|
||||
.flatten(1, 2)
|
||||
.flatten(2, 3)
|
||||
.flatten(3, 4)
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
"""Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head)."""
|
||||
|
||||
@@ -179,39 +115,6 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None:
|
||||
if tp_size <= 0:
|
||||
raise ValueError("Kandinsky6SR TP size must be positive.")
|
||||
for name, value in (
|
||||
("num_attention_heads", arch.num_attention_heads),
|
||||
("ff_dim", arch.ff_dim),
|
||||
):
|
||||
if value % tp_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_sequence_parallel_config(
|
||||
*, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int
|
||||
) -> None:
|
||||
if ulysses_size <= 0:
|
||||
raise ValueError("Kandinsky6SR Ulysses size must be positive.")
|
||||
if ring_size <= 0:
|
||||
raise ValueError("Kandinsky6SR ring size must be positive.")
|
||||
if ulysses_size == 1 and ring_size == 1:
|
||||
return
|
||||
# Ring Attention rotates whole K/V shards between ranks rather than
|
||||
# splitting heads, so only Ulysses constrains head divisibility.
|
||||
local_heads = arch.num_attention_heads // tp_size
|
||||
if local_heads % ulysses_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR TP-local attention heads {local_heads} must be "
|
||||
f"divisible by Ulysses size {ulysses_size} (total heads="
|
||||
f"{arch.num_attention_heads}, TP={tp_size})."
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Kandinsky6SRDitConfig,
|
||||
@@ -223,45 +126,19 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
self.quant_config = quant_config
|
||||
head_dim = sum(arch.axes_dims)
|
||||
|
||||
tp_size = get_tp_world_size()
|
||||
ulysses_size, _ = get_ulysses_ctx()
|
||||
ring_size, _ = get_ring_ctx()
|
||||
self._validate_tp_config(arch=arch, tp_size=tp_size)
|
||||
self._validate_sequence_parallel_config(
|
||||
arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size
|
||||
)
|
||||
_validate_parallelism(arch.num_attention_heads, ff_dim=arch.ff_dim)
|
||||
|
||||
self.in_visual_dim = arch.in_visual_dim
|
||||
self.base_out_visual_dim = arch.base_out_visual_dim
|
||||
self.n_grid = arch.n_grid
|
||||
self.model_dim = arch.model_dim
|
||||
self.patch_size = arch.patch_size
|
||||
self.use_motion_score = arch.use_motion_score
|
||||
self.use_lq_noise_cond = arch.use_lq_noise_cond
|
||||
# The input width is fixed by the *trained* configuration ...
|
||||
# overrides change inference behavior, never the trained input width
|
||||
self.visual_embed_dim = (
|
||||
2 * arch.in_visual_dim + 1
|
||||
if arch.trained_wide_input
|
||||
else arch.in_visual_dim
|
||||
)
|
||||
self._build_layers(arch, head_dim=head_dim, quant_config=quant_config)
|
||||
|
||||
self.hidden_size = arch.hidden_size
|
||||
self.num_attention_heads = arch.num_attention_heads
|
||||
self.num_channels_latents = arch.num_channels_latents
|
||||
self.layer_names = list(_BLOCK_CONTAINERS)
|
||||
# ... while inference behaviour comes from the post-load overrides.
|
||||
self._apply_attribute_overrides(arch)
|
||||
self._require_all_checkpoint_keys()
|
||||
self.__post_init__()
|
||||
|
||||
def _build_layers(
|
||||
self,
|
||||
arch: Kandinsky6SRArchConfig,
|
||||
*,
|
||||
head_dim: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> None:
|
||||
self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim)
|
||||
if arch.use_motion_score:
|
||||
self.motion_embeddings = Kandinsky6TimeEmbeddings(
|
||||
@@ -295,35 +172,17 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
for i in range(arch.num_visual_blocks)
|
||||
]
|
||||
)
|
||||
self.out_layer = Kandinsky6SROutLayer(
|
||||
self.out_layer = Kandinsky6OutLayer(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.head_width,
|
||||
arch.out_visual_dim,
|
||||
arch.patch_size,
|
||||
)
|
||||
|
||||
def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None:
|
||||
self.instruct_type = arch.instruct_type
|
||||
self.visual_cond = arch.visual_cond
|
||||
self.attention_params = arch.attention_params
|
||||
for name in ATTRIBUTE_OVERRIDE_WHITELIST:
|
||||
if name in arch.attribute_overrides:
|
||||
setattr(self, name, arch.attribute_overrides[name])
|
||||
self._check_input_width_matches_overrides()
|
||||
sparse_type = arch.requested_sparse_attention()
|
||||
if sparse_type is not None:
|
||||
logger.warning(
|
||||
"Kandinsky6SR: attention_params request %r sparse attention; this port "
|
||||
"runs dense attention, so outputs differ from the sparse reference.",
|
||||
sparse_type,
|
||||
)
|
||||
|
||||
def _check_input_width_matches_overrides(self) -> None:
|
||||
run_wide = self.visual_cond or self.instruct_type in (
|
||||
"channel",
|
||||
"hybrid",
|
||||
"hybrid_anchor",
|
||||
)
|
||||
self.instruct_type = arch.effective_override("instruct_type")
|
||||
self.visual_cond = arch.effective_override("visual_cond")
|
||||
self.attention_params = arch.effective_override("attention_params")
|
||||
run_wide = self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES
|
||||
run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim
|
||||
if run_width != self.visual_embed_dim:
|
||||
raise ValueError(
|
||||
@@ -334,10 +193,21 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
"under instruct_type='noise'."
|
||||
)
|
||||
|
||||
def _require_all_checkpoint_keys(self) -> None:
|
||||
"""A missing tensor must fail loudly instead of being zero-filled by the loader."""
|
||||
sparse_type = arch.requested_sparse_attention()
|
||||
if sparse_type is not None:
|
||||
logger.warning(
|
||||
"Kandinsky6SR: attention_params request %r sparse attention; this port "
|
||||
"runs dense attention, so outputs differ from the sparse reference.",
|
||||
sparse_type,
|
||||
)
|
||||
self.hidden_size = arch.hidden_size
|
||||
self.num_attention_heads = arch.num_attention_heads
|
||||
self.num_channels_latents = arch.num_channels_latents
|
||||
self.layer_names = list(_BLOCK_CONTAINERS)
|
||||
# missing checkpoint tensors must fail, not be zero-filled
|
||||
for param in self.parameters():
|
||||
param.missing_param_init = "error"
|
||||
self.__post_init__()
|
||||
|
||||
def preprocess_loaded_state_dict(
|
||||
self, weight_iterator: Iterable[tuple[str, torch.Tensor]]
|
||||
@@ -371,7 +241,16 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
timesteps are scaled by 1000. DX channels are ordered grid-major;
|
||||
the scheduler reshapes them into n_grid velocity predictions."""
|
||||
self._check_latent_shape(hidden_states)
|
||||
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR expects a [B, T, H, W, "
|
||||
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
|
||||
)
|
||||
height, width = hidden_states.shape[2:4]
|
||||
if height % self.patch_size[1] or width % self.patch_size[2]:
|
||||
raise ValueError(
|
||||
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
|
||||
)
|
||||
compute_dtype = self.visual_embeddings.in_layer.weight.dtype
|
||||
time_embed = self.time_embeddings(timestep)
|
||||
if motion_score is not None and self.use_motion_score:
|
||||
@@ -397,19 +276,11 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
attn_mask_meta=attn_meta,
|
||||
)
|
||||
visual_embed = gather_seq(visual_embed, shard.orig_len)
|
||||
return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed)
|
||||
|
||||
def _check_latent_shape(self, hidden_states: torch.Tensor) -> None:
|
||||
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR expects a [B, T, H, W, "
|
||||
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
|
||||
)
|
||||
height, width = hidden_states.shape[2:4]
|
||||
if height % self.patch_size[1] or width % self.patch_size[2]:
|
||||
raise ValueError(
|
||||
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
|
||||
)
|
||||
return self.out_layer(
|
||||
visual_embed.reshape(*visual_shape, -1),
|
||||
time_embed,
|
||||
compute_dtype=self.out_layer.out_layer.weight.dtype,
|
||||
)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
"""Rebuild RoPE / time-embedding tables that a meta-device init left on meta."""
|
||||
|
||||
@@ -25,119 +25,10 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler
|
||||
)
|
||||
|
||||
|
||||
class DXPolicy:
|
||||
def __init__( # noqa: PLR0913
|
||||
self,
|
||||
denoising_output: torch.Tensor,
|
||||
x_t_src: torch.Tensor,
|
||||
sigma_t_src: torch.Tensor,
|
||||
segment_size: float | torch.Tensor = 1.0,
|
||||
shift: float = 1.0,
|
||||
mode: str = "grid",
|
||||
eps: float = 1e-4,
|
||||
) -> None:
|
||||
self.x_t_src = x_t_src
|
||||
self.ndim = x_t_src.dim()
|
||||
self.shift = shift
|
||||
self.eps = eps
|
||||
if mode not in ("grid", "polynomial"):
|
||||
raise ValueError(f"Unknown mode: {mode}")
|
||||
self.mode = mode
|
||||
|
||||
self.sigma_t_src = sigma_t_src.reshape(
|
||||
*sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1])
|
||||
)
|
||||
self.raw_t_src = self._unwarp_t(self.sigma_t_src)
|
||||
segment = segment_size
|
||||
if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim():
|
||||
segment = segment.reshape(
|
||||
*segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1])
|
||||
)
|
||||
self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0)
|
||||
self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps)
|
||||
self.denoising_output_x_0 = self._u_to_x_0(
|
||||
denoising_output, self.x_t_src, self.sigma_t_src
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
|
||||
n = x.size(1)
|
||||
if n < 2: # noqa: PLR2004
|
||||
return x.squeeze(1)
|
||||
t = t.clamp(min=0, max=1) * (n - 1)
|
||||
t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2)
|
||||
t1 = t0 + 1
|
||||
indices = torch.stack([t0, t1], dim=1)
|
||||
values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:]))
|
||||
return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
|
||||
|
||||
def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor:
|
||||
return sigma_t / (self.shift + (1 - self.shift) * sigma_t)
|
||||
|
||||
@staticmethod
|
||||
def _u_to_x_0(
|
||||
denoising_output: torch.Tensor,
|
||||
x_t: torch.Tensor,
|
||||
sigma_t: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output
|
||||
|
||||
def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor:
|
||||
sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1]))
|
||||
raw_t = self._unwarp_t(sigma_t)
|
||||
if self.mode == "grid":
|
||||
x_0 = self._interpolate(
|
||||
self.denoising_output_x_0,
|
||||
(raw_t - self.raw_t_dst) / self.segment_size,
|
||||
)
|
||||
else:
|
||||
p_order = self.denoising_output_x_0.size(1)
|
||||
diff_t = self.raw_t_src - raw_t
|
||||
basis = torch.stack([diff_t**i for i in range(p_order)], dim=1)
|
||||
x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1)
|
||||
return (x_t - x_0) / sigma_t.clamp(min=self.eps)
|
||||
|
||||
|
||||
def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor:
|
||||
return shift * t / (1 + (shift - 1) * t)
|
||||
|
||||
|
||||
def policy_rollout_fm( # noqa: PLR0913
|
||||
x_t_start: torch.Tensor,
|
||||
sigma_t_start: torch.Tensor,
|
||||
raw_t_start: torch.Tensor,
|
||||
raw_t_end: torch.Tensor,
|
||||
total_substeps: int,
|
||||
policy: DXPolicy,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
num_batches = x_t_start.size(0)
|
||||
ndim = x_t_start.dim()
|
||||
shape = (num_batches, *((ndim - 1) * [1]))
|
||||
raw_t_start = raw_t_start.reshape(shape)
|
||||
raw_t_end = raw_t_end.reshape(shape)
|
||||
sigma_t = sigma_t_start.reshape(shape)
|
||||
|
||||
delta_raw_t = raw_t_start - raw_t_end
|
||||
num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1)
|
||||
substep_size = delta_raw_t / num_substeps
|
||||
max_num_substeps = num_substeps.max()
|
||||
|
||||
raw_t = raw_t_start
|
||||
x_t = x_t_start
|
||||
for substep_id in range(max_num_substeps.item()):
|
||||
velocity = policy.pi(x_t, sigma_t)
|
||||
raw_t_minus = (raw_t - substep_size).clamp(min=0)
|
||||
sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift)
|
||||
x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t)
|
||||
|
||||
active_mask = num_substeps > substep_id
|
||||
x_t = torch.where(active_mask, x_t_minus, x_t)
|
||||
sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t)
|
||||
raw_t = torch.where(active_mask, raw_t_minus, raw_t)
|
||||
|
||||
return x_t, sigma_t, sigma_t.flatten() * 1_000
|
||||
|
||||
|
||||
class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
|
||||
is_piflow = True
|
||||
|
||||
@@ -239,25 +130,41 @@ class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
|
||||
sigma_src = self.sigmas[step_index].to(device=sample.device)
|
||||
token_shape = (sample.shape[0], *((sample.ndim - 1) * [1]))
|
||||
sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape)
|
||||
segment = (raw_src - raw_dst).expand(sample.shape[0])
|
||||
policy = DXPolicy(
|
||||
model_output,
|
||||
sample,
|
||||
sigma,
|
||||
segment,
|
||||
shift=float(self.config.shift),
|
||||
mode="grid",
|
||||
eps=self.eps,
|
||||
)
|
||||
updated, _, _ = policy_rollout_fm(
|
||||
sample,
|
||||
sigma,
|
||||
raw_src.expand(sample.shape[0]),
|
||||
raw_dst.expand(sample.shape[0]),
|
||||
self.num_policy_substeps,
|
||||
policy,
|
||||
)
|
||||
return updated.to(dtype=sample.dtype)
|
||||
segment = (raw_src - raw_dst).expand(sample.shape[0]).reshape(token_shape)
|
||||
shift = float(self.config.shift)
|
||||
policy_src = sigma / (shift + (1 - shift) * sigma)
|
||||
policy_dst = (policy_src - segment).clamp(min=0)
|
||||
policy_segment = (policy_src - policy_dst).clamp(min=self.eps)
|
||||
x0_grid = sample.unsqueeze(1) - sigma.unsqueeze(1) * model_output
|
||||
|
||||
raw_t = raw_src.expand(sample.shape[0]).reshape(token_shape)
|
||||
delta = raw_t - raw_dst.expand(sample.shape[0]).reshape(token_shape)
|
||||
substeps = (delta * self.num_policy_substeps).round().long().clamp(min=1)
|
||||
substep_size = delta / substeps
|
||||
for index in range(substeps.max().item()):
|
||||
# the checkpoint predicts x0 on a grid in unwarped time
|
||||
policy_t = sigma / (shift + (1 - shift) * sigma)
|
||||
t = ((policy_t - policy_dst) / policy_segment).clamp(0, 1) * (
|
||||
self.n_grid - 1
|
||||
)
|
||||
t0 = t.floor().long().clamp(0, self.n_grid - 2)
|
||||
t1 = t0 + 1
|
||||
indices = torch.stack([t0, t1], dim=1)
|
||||
values = torch.gather(
|
||||
x0_grid, 1, indices.expand(-1, -1, *x0_grid.shape[2:])
|
||||
)
|
||||
x0 = (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
|
||||
velocity = (sample - x0) / sigma.clamp(min=self.eps)
|
||||
# release grid interpolation temporaries before the state update/next gather
|
||||
del values, x0, indices, t, t0, t1, policy_t
|
||||
next_raw = (raw_t - substep_size).clamp(min=0)
|
||||
next_sigma = shift_timesteps(next_raw, shift)
|
||||
updated = sample + velocity * (next_sigma - sigma)
|
||||
active = substeps > index
|
||||
sample = torch.where(active, updated, sample)
|
||||
sigma = torch.where(active, next_sigma, sigma)
|
||||
raw_t = torch.where(active, next_raw, raw_t)
|
||||
return sample
|
||||
|
||||
def _step_index_for(self, timestep: torch.Tensor | float) -> int:
|
||||
if self.step_index is None:
|
||||
|
||||
+20
-51
@@ -48,9 +48,7 @@ class RMSNorm(nn.Module):
|
||||
self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1))
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
# Upcast to float32 around the normalize (matches the Diffusers reference's
|
||||
# Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the
|
||||
# reference across the many ResidualBlock/X2Branch calls that route through this norm.
|
||||
# preserve the reference's fp32 reduction followed by bf16 scale/gain
|
||||
normalized = F.normalize(x.float(), dim=1).to(x.dtype)
|
||||
return normalized * self.scale * self.gamma
|
||||
|
||||
@@ -96,31 +94,8 @@ class ResidualBlock(nn.Module):
|
||||
return self.shortcut(x) + h
|
||||
|
||||
|
||||
def nearest_2x(x: Tensor) -> Tensor:
|
||||
"""Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch)."""
|
||||
b, c, t, h, w = x.shape
|
||||
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
|
||||
x = F.interpolate(x, scale_factor=2, mode="nearest")
|
||||
return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
|
||||
|
||||
|
||||
class PXSUpsample(nn.Module):
|
||||
"""2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``."""
|
||||
|
||||
def __init__(self, channels: int) -> None:
|
||||
super().__init__()
|
||||
self.spatial_conv = nn.Conv3d(
|
||||
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
|
||||
)
|
||||
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
up = nearest_2x(x)
|
||||
return self.linear(up + self.spatial_conv(up))
|
||||
|
||||
|
||||
class X2Finisher(nn.Module):
|
||||
"""The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize)."""
|
||||
"""Residual spatial convolution and channel projection, without resizing."""
|
||||
|
||||
def __init__(self, channels: int) -> None:
|
||||
super().__init__()
|
||||
@@ -133,19 +108,21 @@ class X2Finisher(nn.Module):
|
||||
return self.linear(x + self.spatial_conv(x))
|
||||
|
||||
|
||||
class PXSUpsample(X2Finisher):
|
||||
"""Apply the same projection after nearest-neighbor 2x spatial resizing."""
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
b, c, t, h, w = x.shape
|
||||
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
|
||||
x = F.interpolate(x, scale_factor=2, mode="nearest")
|
||||
x = x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
|
||||
return super().forward(x)
|
||||
|
||||
|
||||
def _stem(in_channels: int, out_channels: int) -> nn.Sequential:
|
||||
return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3))
|
||||
|
||||
|
||||
def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential:
|
||||
"""Unused training head whose checkpoint layout remains positional."""
|
||||
return nn.Sequential(
|
||||
RMSNorm(channels),
|
||||
nn.SiLU(),
|
||||
ReplicateTimeConv3d(channels, out_channels, kernel_size=3),
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscalerOutputHead(nn.Module):
|
||||
"""Named ``norm -> activation -> conv`` projection used by current checkpoints."""
|
||||
|
||||
@@ -182,18 +159,6 @@ def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor:
|
||||
return x
|
||||
|
||||
|
||||
def _second_stage(
|
||||
upsample: nn.Module,
|
||||
blocks: nn.Sequential,
|
||||
head: Kandinsky6SRLatentUpscalerOutputHead,
|
||||
x: Tensor,
|
||||
zq: Tensor,
|
||||
) -> Tensor:
|
||||
"""``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``."""
|
||||
x = _run_blocks(blocks, upsample(x), zq)
|
||||
return head(x, zq)
|
||||
|
||||
|
||||
class X2Branch(nn.Module):
|
||||
"""Weights exclusive to the x2 path."""
|
||||
|
||||
@@ -213,7 +178,8 @@ class X2Branch(nn.Module):
|
||||
x = _run_blocks(self.adapter, x, zq)
|
||||
x = self.finisher(x)
|
||||
x = _run_blocks(self.mid_blocks, x, zq)
|
||||
return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq)
|
||||
x = _run_blocks(self.blocks, self.upsample(x), zq)
|
||||
return self.output_proj(x, zq)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscaler(nn.Module):
|
||||
@@ -227,7 +193,9 @@ class Kandinsky6SRLatentUpscaler(nn.Module):
|
||||
er = config.expand_ratio
|
||||
self.input_proj = _stem(c, w1)
|
||||
# The 2x deep-supervision head of training: part of the checkpoint, never run at inference.
|
||||
self.mid_output_head = _mid_output_head(w2, c)
|
||||
self.mid_output_head = nn.Sequential(
|
||||
RMSNorm(w2), nn.SiLU(), ReplicateTimeConv3d(w2, c, kernel_size=3)
|
||||
)
|
||||
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
|
||||
self.upsample_1 = PXSUpsample(w1)
|
||||
self.upsample_2 = PXSUpsample(w2)
|
||||
@@ -254,7 +222,8 @@ class Kandinsky6SRLatentUpscaler(nn.Module):
|
||||
)
|
||||
x = _run_blocks(self.pre_blocks, self.input_proj(z), zq)
|
||||
x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq)
|
||||
return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq)
|
||||
x = _run_blocks(self.post_blocks, self.upsample_2(x), zq)
|
||||
return self.output_proj(x, zq)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
@@ -48,24 +48,13 @@ class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
# audio latent stride; frame-count alignment is owned by the pipeline config
|
||||
self.downsample_factor = 1024
|
||||
|
||||
self.mel_converter = MelConverter(
|
||||
sampling_rate=44100,
|
||||
n_fft=2048,
|
||||
num_mels=128,
|
||||
hop_size=512,
|
||||
win_size=2048,
|
||||
fmin=0,
|
||||
fmax=22050,
|
||||
)
|
||||
self.mel_converter = MelConverter()
|
||||
# checkpoint weights include the encoder regardless of need_vae_encoder
|
||||
self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True)
|
||||
# normalize random initialization before loading; checkpoint weights are
|
||||
# already normalized and must never be renormalized
|
||||
self.vae.remove_weight_norm()
|
||||
self.vocoder = BigVGANV2(dict(arch.vocoder_config))
|
||||
# strip PyTorch weight_norm before loading plain .weight checkpoint keys;
|
||||
# this differs from MMAudio's RMS renormalization above
|
||||
self.vocoder.remove_weight_norm()
|
||||
|
||||
# resolution containers are not called; hooks belong on their blocks
|
||||
self.layer_names = (
|
||||
|
||||
@@ -1,350 +1,141 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright (c) 2024 NVIDIA CORPORATION.
|
||||
# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's
|
||||
# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is
|
||||
# intentionally excluded; this module uses the portable PyTorch path for
|
||||
# deterministic loading and parity across platforms.
|
||||
# Adapted from NVIDIA BigVGAN-v2 (MIT) via FastVideo.
|
||||
# Original license terms are retained in THIRD_PARTY_LICENSES.
|
||||
#
|
||||
# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/
|
||||
# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than
|
||||
# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` --
|
||||
# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and
|
||||
# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more),
|
||||
# while this implementation supports either resblock variant or activation. The
|
||||
# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/
|
||||
# Activation1d) ARE architecture-agnostic DSP building blocks shared with
|
||||
# that same sibling module, though -- both import ``Activation1d`` from
|
||||
# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside
|
||||
# ``common.py``) instead of each keeping its own copy.
|
||||
|
||||
from __future__ import annotations
|
||||
"""BigVGAN-v2 with the plain weights stored in Kandinsky audio checkpoints."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn import Conv1d, ConvTranspose1d
|
||||
from torch.nn.utils.parametrizations import weight_norm
|
||||
from torch.nn.utils.parametrize import remove_parametrizations
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
|
||||
|
||||
|
||||
class AttrDict(dict):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__dict__ = self
|
||||
|
||||
|
||||
def get_padding(kernel_size: int, dilation: int = 1) -> int:
|
||||
return int((kernel_size * dilation - dilation) / 2)
|
||||
|
||||
|
||||
def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None:
|
||||
if "Conv" in module.__class__.__name__:
|
||||
module.weight.data.normal_(mean, std)
|
||||
|
||||
|
||||
class Snake(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
alpha: float = 1.0,
|
||||
alpha_trainable: bool = True,
|
||||
alpha_logscale: bool = False,
|
||||
) -> None:
|
||||
def __init__(self, channels: int, *, logscale: bool, separate_beta: bool):
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.alpha_logscale = alpha_logscale
|
||||
initial = (
|
||||
torch.zeros(in_features) * alpha
|
||||
if alpha_logscale
|
||||
else torch.ones(in_features) * alpha
|
||||
)
|
||||
self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable)
|
||||
self.no_div_by_zero = 1e-9
|
||||
initial = torch.zeros(channels) if logscale else torch.ones(channels)
|
||||
self.alpha = nn.Parameter(initial)
|
||||
self.beta = nn.Parameter(initial.clone()) if separate_beta else None
|
||||
self.logscale = logscale
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
||||
if self.alpha_logscale:
|
||||
alpha = torch.exp(alpha)
|
||||
return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow(
|
||||
torch.sin(x * alpha), 2
|
||||
alpha = self.alpha[None, :, None]
|
||||
beta = alpha if self.beta is None else self.beta[None, :, None]
|
||||
if self.logscale:
|
||||
alpha = alpha.exp()
|
||||
beta = alpha if self.beta is None else beta.exp()
|
||||
return x + (1.0 / (beta + 1e-9)) * torch.sin(x * alpha).pow(2)
|
||||
|
||||
|
||||
def _activation(config: SimpleNamespace, channels: int) -> Activation1d:
|
||||
if config.activation not in ("snake", "snakebeta"):
|
||||
raise ValueError(f"Unsupported BigVGAN activation: {config.activation}")
|
||||
return Activation1d(
|
||||
Snake(
|
||||
channels,
|
||||
logscale=config.snake_logscale,
|
||||
separate_beta=config.activation == "snakebeta",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class SnakeBeta(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
alpha: float = 1.0,
|
||||
alpha_trainable: bool = True,
|
||||
alpha_logscale: bool = False,
|
||||
) -> None:
|
||||
class AMPBlock(nn.Module):
|
||||
def __init__(self, config, channels: int, kernel: int, dilations):
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.alpha_logscale = alpha_logscale
|
||||
initial = (
|
||||
torch.zeros(in_features) * alpha
|
||||
if alpha_logscale
|
||||
else torch.ones(in_features) * alpha
|
||||
self.paired = config.resblock == "1"
|
||||
convs = nn.ModuleList(
|
||||
nn.Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel,
|
||||
dilation=rate,
|
||||
padding=(kernel - 1) * rate // 2,
|
||||
)
|
||||
for rate in dilations
|
||||
)
|
||||
self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
|
||||
self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
|
||||
self.no_div_by_zero = 1e-9
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
||||
beta = self.beta.unsqueeze(0).unsqueeze(-1)
|
||||
if self.alpha_logscale:
|
||||
alpha = torch.exp(alpha)
|
||||
beta = torch.exp(beta)
|
||||
return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
|
||||
torch.sin(x * alpha), 2
|
||||
)
|
||||
|
||||
|
||||
def _activation(name: str, channels: int, logscale: bool) -> Activation1d:
|
||||
if name == "snake":
|
||||
activation = Snake(channels, alpha_logscale=logscale)
|
||||
elif name == "snakebeta":
|
||||
activation = SnakeBeta(channels, alpha_logscale=logscale)
|
||||
else:
|
||||
raise ValueError(f"Unsupported BigVGAN activation: {name}")
|
||||
return Activation1d(activation)
|
||||
|
||||
|
||||
class AMPBlock1(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AttrDict,
|
||||
channels: int,
|
||||
kernel_size: int = 3,
|
||||
dilation: tuple[int, ...] = (1, 3, 5),
|
||||
activation: str = "snake",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.convs1 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=rate,
|
||||
padding=get_padding(kernel_size, rate),
|
||||
)
|
||||
)
|
||||
for rate in dilation
|
||||
]
|
||||
)
|
||||
self.convs1.apply(init_weights)
|
||||
self.convs2 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=1,
|
||||
padding=get_padding(kernel_size, 1),
|
||||
)
|
||||
)
|
||||
for _ in dilation
|
||||
]
|
||||
)
|
||||
self.convs2.apply(init_weights)
|
||||
# retain both published state-dict layouts
|
||||
if self.paired:
|
||||
self.convs1 = convs
|
||||
self.convs2 = nn.ModuleList(
|
||||
nn.Conv1d(channels, channels, kernel, padding=(kernel - 1) // 2)
|
||||
for _ in dilations
|
||||
)
|
||||
else:
|
||||
self.convs = convs
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
_activation(activation, channels, config.snake_logscale)
|
||||
for _ in range(len(self.convs1) + len(self.convs2))
|
||||
]
|
||||
_activation(config, channels)
|
||||
for _ in range(len(dilations) * (2 if self.paired else 1))
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
acts1, acts2 = self.activations[::2], self.activations[1::2]
|
||||
for conv1, conv2, act1, act2 in zip(
|
||||
self.convs1, self.convs2, acts1, acts2, strict=True
|
||||
):
|
||||
residual = conv2(act2(conv1(act1(x))))
|
||||
x = residual + x
|
||||
if self.paired:
|
||||
for index, (conv1, conv2) in enumerate(zip(self.convs1, self.convs2)):
|
||||
hidden = conv1(self.activations[2 * index](x))
|
||||
x = conv2(self.activations[2 * index + 1](hidden)) + x
|
||||
else:
|
||||
for conv, activation in zip(self.convs, self.activations):
|
||||
x = conv(activation(x)) + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
for layer in self.convs1:
|
||||
remove_parametrizations(layer, "weight")
|
||||
for layer in self.convs2:
|
||||
remove_parametrizations(layer, "weight")
|
||||
|
||||
|
||||
class AMPBlock2(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AttrDict,
|
||||
channels: int,
|
||||
kernel_size: int = 3,
|
||||
dilation: tuple[int, ...] = (1, 3, 5),
|
||||
activation: str = "snake",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=rate,
|
||||
padding=get_padding(kernel_size, rate),
|
||||
)
|
||||
)
|
||||
for rate in dilation
|
||||
]
|
||||
)
|
||||
self.convs.apply(init_weights)
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
_activation(activation, channels, config.snake_logscale)
|
||||
for _ in self.convs
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
for conv, activation in zip(self.convs, self.activations, strict=True):
|
||||
x = conv(activation(x)) + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
for layer in self.convs:
|
||||
remove_parametrizations(layer, "weight")
|
||||
|
||||
|
||||
class BigVGANV2(nn.Module):
|
||||
"""BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path."""
|
||||
|
||||
def __init__(self, config: dict[str, Any]) -> None:
|
||||
super().__init__()
|
||||
config = dict(config)
|
||||
config.pop("_class_name", None)
|
||||
config.pop("architectures", None)
|
||||
weight_norm_removed = bool(config.pop("weight_norm_removed", False))
|
||||
self.config = AttrDict(config)
|
||||
if self.config.get("use_cuda_kernel", False):
|
||||
if config.get("use_cuda_kernel", False):
|
||||
raise ValueError(
|
||||
"Kandinsky6 BigVGANV2 supports only the portable PyTorch path"
|
||||
)
|
||||
self.config["use_cuda_kernel"] = False
|
||||
self.num_kernels = len(self.config.resblock_kernel_sizes)
|
||||
self.num_upsamples = len(self.config.upsample_rates)
|
||||
self.conv_pre = weight_norm(
|
||||
Conv1d(
|
||||
self.config.num_mels,
|
||||
self.config.upsample_initial_channel,
|
||||
7,
|
||||
1,
|
||||
padding=3,
|
||||
)
|
||||
)
|
||||
if self.config.resblock == "1":
|
||||
block_class = AMPBlock1
|
||||
elif self.config.resblock == "2":
|
||||
block_class = AMPBlock2
|
||||
else:
|
||||
raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}")
|
||||
|
||||
h = SimpleNamespace(**config)
|
||||
if h.resblock not in ("1", "2"):
|
||||
raise ValueError(f"Unsupported BigVGAN resblock: {h.resblock}")
|
||||
self.num_kernels = len(h.resblock_kernel_sizes)
|
||||
self.conv_pre = nn.Conv1d(h.num_mels, h.upsample_initial_channel, 7, padding=3)
|
||||
self.ups = nn.ModuleList()
|
||||
for index, (rate, kernel) in enumerate(
|
||||
zip(
|
||||
self.config.upsample_rates,
|
||||
self.config.upsample_kernel_sizes,
|
||||
strict=True,
|
||||
)
|
||||
):
|
||||
self.resblocks = nn.ModuleList()
|
||||
channels = h.upsample_initial_channel
|
||||
for rate, kernel in zip(h.upsample_rates, h.upsample_kernel_sizes, strict=True):
|
||||
self.ups.append(
|
||||
nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
ConvTranspose1d(
|
||||
self.config.upsample_initial_channel // (2**index),
|
||||
self.config.upsample_initial_channel
|
||||
// (2 ** (index + 1)),
|
||||
kernel,
|
||||
rate,
|
||||
padding=(kernel - rate) // 2,
|
||||
)
|
||||
nn.ConvTranspose1d(
|
||||
channels,
|
||||
channels // 2,
|
||||
kernel,
|
||||
rate,
|
||||
padding=(kernel - rate) // 2,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
for index in range(len(self.ups)):
|
||||
channels = self.config.upsample_initial_channel // (2 ** (index + 1))
|
||||
for kernel, dilation in zip(
|
||||
self.config.resblock_kernel_sizes,
|
||||
self.config.resblock_dilation_sizes,
|
||||
strict=True,
|
||||
):
|
||||
self.resblocks.append(
|
||||
block_class(
|
||||
self.config,
|
||||
channels,
|
||||
kernel,
|
||||
tuple(dilation),
|
||||
activation=self.config.activation,
|
||||
)
|
||||
channels //= 2
|
||||
self.resblocks.extend(
|
||||
AMPBlock(h, channels, kernel, dilations)
|
||||
for kernel, dilations in zip(
|
||||
h.resblock_kernel_sizes, h.resblock_dilation_sizes, strict=True
|
||||
)
|
||||
|
||||
channels = self.config.upsample_initial_channel // (2 ** len(self.ups))
|
||||
self.activation_post = _activation(
|
||||
self.config.activation, channels, self.config.snake_logscale
|
||||
)
|
||||
self.activation_post = _activation(h, channels)
|
||||
self.conv_post = nn.Conv1d(
|
||||
channels, 1, 7, padding=3, bias=config.get("use_bias_at_final", True)
|
||||
)
|
||||
self.use_bias_at_final = self.config.get("use_bias_at_final", True)
|
||||
self.conv_post = weight_norm(
|
||||
Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final)
|
||||
)
|
||||
for upsampler in self.ups:
|
||||
upsampler.apply(init_weights)
|
||||
self.conv_post.apply(init_weights)
|
||||
self.use_tanh_at_final = self.config.get("use_tanh_at_final", True)
|
||||
if weight_norm_removed:
|
||||
self.remove_weight_norm()
|
||||
self.use_tanh_at_final = config.get("use_tanh_at_final", True)
|
||||
|
||||
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
||||
hidden = self.conv_pre(mel)
|
||||
for index in range(self.num_upsamples):
|
||||
for upsampler in self.ups[index]:
|
||||
for index, upsamplers in enumerate(self.ups):
|
||||
for upsampler in upsamplers:
|
||||
hidden = upsampler(hidden)
|
||||
accumulated = None
|
||||
for kernel in range(self.num_kernels):
|
||||
block_output = self.resblocks[index * self.num_kernels + kernel](hidden)
|
||||
output = self.resblocks[index * self.num_kernels + kernel](hidden)
|
||||
if accumulated is None:
|
||||
accumulated = block_output
|
||||
accumulated = output
|
||||
else:
|
||||
# Preserve BigVGAN's published sequential accumulation
|
||||
# order. Tree reduction drifts through later nonlinear
|
||||
# upsampling stages with the full checkpoint.
|
||||
accumulated += block_output
|
||||
assert accumulated is not None
|
||||
# preserve sequential accumulation through nonlinear upsampling
|
||||
accumulated += output
|
||||
hidden = accumulated / self.num_kernels
|
||||
hidden = self.conv_post(self.activation_post(hidden))
|
||||
if self.use_tanh_at_final:
|
||||
return torch.tanh(hidden)
|
||||
return torch.clamp(hidden, min=-1.0, max=1.0)
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
try:
|
||||
for upsamplers in self.ups:
|
||||
for upsampler in upsamplers:
|
||||
remove_parametrizations(upsampler, "weight")
|
||||
for block in self.resblocks:
|
||||
block.remove_weight_norm()
|
||||
remove_parametrizations(self.conv_pre, "weight")
|
||||
remove_parametrizations(self.conv_post, "weight")
|
||||
except ValueError:
|
||||
# Idempotent for pipeline setup and converted checkpoints.
|
||||
return
|
||||
return torch.tanh(hidden) if self.use_tanh_at_final else hidden.clamp(-1.0, 1.0)
|
||||
|
||||
+75
-573
@@ -1,25 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License,
|
||||
# Copyright (c) 2024 Sony Research Inc., via FastVideo's
|
||||
# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram
|
||||
# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae``
|
||||
# (see kandinsky6_audio.py in the parent directory).
|
||||
# Copyright (c) 2024 Sony Research Inc., via FastVideo.
|
||||
# Original license terms are retained in THIRD_PARTY_LICENSES.
|
||||
#
|
||||
# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its
|
||||
# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is
|
||||
# dropped -- sglang's generic VAELoader never calls a per-submodule
|
||||
# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict``
|
||||
# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code
|
||||
# here.
|
||||
"""Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from typing import Any
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -29,435 +19,6 @@ from einops import rearrange
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DATA_MEAN_80D = [
|
||||
-1.6058,
|
||||
-1.3676,
|
||||
-1.2520,
|
||||
-1.2453,
|
||||
-1.2078,
|
||||
-1.2224,
|
||||
-1.2419,
|
||||
-1.2439,
|
||||
-1.2922,
|
||||
-1.2927,
|
||||
-1.3170,
|
||||
-1.3543,
|
||||
-1.3401,
|
||||
-1.3836,
|
||||
-1.3907,
|
||||
-1.3912,
|
||||
-1.4313,
|
||||
-1.4152,
|
||||
-1.4527,
|
||||
-1.4728,
|
||||
-1.4568,
|
||||
-1.5101,
|
||||
-1.5051,
|
||||
-1.5172,
|
||||
-1.5623,
|
||||
-1.5373,
|
||||
-1.5746,
|
||||
-1.5687,
|
||||
-1.6032,
|
||||
-1.6131,
|
||||
-1.6081,
|
||||
-1.6331,
|
||||
-1.6489,
|
||||
-1.6489,
|
||||
-1.6700,
|
||||
-1.6738,
|
||||
-1.6953,
|
||||
-1.6969,
|
||||
-1.7048,
|
||||
-1.7280,
|
||||
-1.7361,
|
||||
-1.7495,
|
||||
-1.7658,
|
||||
-1.7814,
|
||||
-1.7889,
|
||||
-1.8064,
|
||||
-1.8221,
|
||||
-1.8377,
|
||||
-1.8417,
|
||||
-1.8643,
|
||||
-1.8857,
|
||||
-1.8929,
|
||||
-1.9173,
|
||||
-1.9379,
|
||||
-1.9531,
|
||||
-1.9673,
|
||||
-1.9824,
|
||||
-2.0042,
|
||||
-2.0215,
|
||||
-2.0436,
|
||||
-2.0766,
|
||||
-2.1064,
|
||||
-2.1418,
|
||||
-2.1855,
|
||||
-2.2319,
|
||||
-2.2767,
|
||||
-2.3161,
|
||||
-2.3572,
|
||||
-2.3954,
|
||||
-2.4282,
|
||||
-2.4659,
|
||||
-2.5072,
|
||||
-2.5552,
|
||||
-2.6074,
|
||||
-2.6584,
|
||||
-2.7107,
|
||||
-2.7634,
|
||||
-2.8266,
|
||||
-2.8981,
|
||||
-2.9673,
|
||||
]
|
||||
|
||||
DATA_STD_80D = [
|
||||
1.0291,
|
||||
1.0411,
|
||||
1.0043,
|
||||
0.9820,
|
||||
0.9677,
|
||||
0.9543,
|
||||
0.9450,
|
||||
0.9392,
|
||||
0.9343,
|
||||
0.9297,
|
||||
0.9276,
|
||||
0.9263,
|
||||
0.9242,
|
||||
0.9254,
|
||||
0.9232,
|
||||
0.9281,
|
||||
0.9263,
|
||||
0.9315,
|
||||
0.9274,
|
||||
0.9247,
|
||||
0.9277,
|
||||
0.9199,
|
||||
0.9188,
|
||||
0.9194,
|
||||
0.9160,
|
||||
0.9161,
|
||||
0.9146,
|
||||
0.9161,
|
||||
0.9100,
|
||||
0.9095,
|
||||
0.9145,
|
||||
0.9076,
|
||||
0.9066,
|
||||
0.9095,
|
||||
0.9032,
|
||||
0.9043,
|
||||
0.9038,
|
||||
0.9011,
|
||||
0.9019,
|
||||
0.9010,
|
||||
0.8984,
|
||||
0.8983,
|
||||
0.8986,
|
||||
0.8961,
|
||||
0.8962,
|
||||
0.8978,
|
||||
0.8962,
|
||||
0.8973,
|
||||
0.8993,
|
||||
0.8976,
|
||||
0.8995,
|
||||
0.9016,
|
||||
0.8982,
|
||||
0.8972,
|
||||
0.8974,
|
||||
0.8949,
|
||||
0.8940,
|
||||
0.8947,
|
||||
0.8936,
|
||||
0.8939,
|
||||
0.8951,
|
||||
0.8956,
|
||||
0.9017,
|
||||
0.9167,
|
||||
0.9436,
|
||||
0.9690,
|
||||
1.0003,
|
||||
1.0225,
|
||||
1.0381,
|
||||
1.0491,
|
||||
1.0545,
|
||||
1.0604,
|
||||
1.0761,
|
||||
1.0929,
|
||||
1.1089,
|
||||
1.1196,
|
||||
1.1176,
|
||||
1.1156,
|
||||
1.1117,
|
||||
1.1070,
|
||||
]
|
||||
|
||||
DATA_MEAN_128D = [
|
||||
-3.3462,
|
||||
-2.6723,
|
||||
-2.4893,
|
||||
-2.3143,
|
||||
-2.2664,
|
||||
-2.3317,
|
||||
-2.1802,
|
||||
-2.4006,
|
||||
-2.2357,
|
||||
-2.4597,
|
||||
-2.3717,
|
||||
-2.4690,
|
||||
-2.5142,
|
||||
-2.4919,
|
||||
-2.6610,
|
||||
-2.5047,
|
||||
-2.7483,
|
||||
-2.5926,
|
||||
-2.7462,
|
||||
-2.7033,
|
||||
-2.7386,
|
||||
-2.8112,
|
||||
-2.7502,
|
||||
-2.9594,
|
||||
-2.7473,
|
||||
-3.0035,
|
||||
-2.8891,
|
||||
-2.9922,
|
||||
-2.9856,
|
||||
-3.0157,
|
||||
-3.1191,
|
||||
-2.9893,
|
||||
-3.1718,
|
||||
-3.0745,
|
||||
-3.1879,
|
||||
-3.2310,
|
||||
-3.1424,
|
||||
-3.2296,
|
||||
-3.2791,
|
||||
-3.2782,
|
||||
-3.2756,
|
||||
-3.3134,
|
||||
-3.3509,
|
||||
-3.3750,
|
||||
-3.3951,
|
||||
-3.3698,
|
||||
-3.4505,
|
||||
-3.4509,
|
||||
-3.5089,
|
||||
-3.4647,
|
||||
-3.5536,
|
||||
-3.5788,
|
||||
-3.5867,
|
||||
-3.6036,
|
||||
-3.6400,
|
||||
-3.6747,
|
||||
-3.7072,
|
||||
-3.7279,
|
||||
-3.7283,
|
||||
-3.7795,
|
||||
-3.8259,
|
||||
-3.8447,
|
||||
-3.8663,
|
||||
-3.9182,
|
||||
-3.9605,
|
||||
-3.9861,
|
||||
-4.0105,
|
||||
-4.0373,
|
||||
-4.0762,
|
||||
-4.1121,
|
||||
-4.1488,
|
||||
-4.1874,
|
||||
-4.2461,
|
||||
-4.3170,
|
||||
-4.3639,
|
||||
-4.4452,
|
||||
-4.5282,
|
||||
-4.6297,
|
||||
-4.7019,
|
||||
-4.7960,
|
||||
-4.8700,
|
||||
-4.9507,
|
||||
-5.0303,
|
||||
-5.0866,
|
||||
-5.1634,
|
||||
-5.2342,
|
||||
-5.3242,
|
||||
-5.4053,
|
||||
-5.4927,
|
||||
-5.5712,
|
||||
-5.6464,
|
||||
-5.7052,
|
||||
-5.7619,
|
||||
-5.8410,
|
||||
-5.9188,
|
||||
-6.0103,
|
||||
-6.0955,
|
||||
-6.1673,
|
||||
-6.2362,
|
||||
-6.3120,
|
||||
-6.3926,
|
||||
-6.4797,
|
||||
-6.5565,
|
||||
-6.6511,
|
||||
-6.8130,
|
||||
-6.9961,
|
||||
-7.1275,
|
||||
-7.2457,
|
||||
-7.3576,
|
||||
-7.4663,
|
||||
-7.6136,
|
||||
-7.7469,
|
||||
-7.8815,
|
||||
-8.0132,
|
||||
-8.1515,
|
||||
-8.3071,
|
||||
-8.4722,
|
||||
-8.7418,
|
||||
-9.3975,
|
||||
-9.6628,
|
||||
-9.7671,
|
||||
-9.8863,
|
||||
-9.9992,
|
||||
-10.0860,
|
||||
-10.1709,
|
||||
-10.5418,
|
||||
-11.2795,
|
||||
-11.3861,
|
||||
]
|
||||
|
||||
DATA_STD_128D = [
|
||||
2.3804,
|
||||
2.4368,
|
||||
2.3772,
|
||||
2.3145,
|
||||
2.2803,
|
||||
2.2510,
|
||||
2.2316,
|
||||
2.2083,
|
||||
2.1996,
|
||||
2.1835,
|
||||
2.1769,
|
||||
2.1659,
|
||||
2.1631,
|
||||
2.1618,
|
||||
2.1540,
|
||||
2.1606,
|
||||
2.1571,
|
||||
2.1567,
|
||||
2.1612,
|
||||
2.1579,
|
||||
2.1679,
|
||||
2.1683,
|
||||
2.1634,
|
||||
2.1557,
|
||||
2.1668,
|
||||
2.1518,
|
||||
2.1415,
|
||||
2.1449,
|
||||
2.1406,
|
||||
2.1350,
|
||||
2.1313,
|
||||
2.1415,
|
||||
2.1281,
|
||||
2.1352,
|
||||
2.1219,
|
||||
2.1182,
|
||||
2.1327,
|
||||
2.1195,
|
||||
2.1137,
|
||||
2.1080,
|
||||
2.1179,
|
||||
2.1036,
|
||||
2.1087,
|
||||
2.1036,
|
||||
2.1015,
|
||||
2.1068,
|
||||
2.0975,
|
||||
2.0991,
|
||||
2.0902,
|
||||
2.1015,
|
||||
2.0857,
|
||||
2.0920,
|
||||
2.0893,
|
||||
2.0897,
|
||||
2.0910,
|
||||
2.0881,
|
||||
2.0925,
|
||||
2.0873,
|
||||
2.0960,
|
||||
2.0900,
|
||||
2.0957,
|
||||
2.0958,
|
||||
2.0978,
|
||||
2.0936,
|
||||
2.0886,
|
||||
2.0905,
|
||||
2.0845,
|
||||
2.0855,
|
||||
2.0796,
|
||||
2.0840,
|
||||
2.0813,
|
||||
2.0817,
|
||||
2.0838,
|
||||
2.0840,
|
||||
2.0917,
|
||||
2.1061,
|
||||
2.1431,
|
||||
2.1976,
|
||||
2.2482,
|
||||
2.3055,
|
||||
2.3700,
|
||||
2.4088,
|
||||
2.4372,
|
||||
2.4609,
|
||||
2.4731,
|
||||
2.4847,
|
||||
2.5072,
|
||||
2.5451,
|
||||
2.5772,
|
||||
2.6147,
|
||||
2.6529,
|
||||
2.6596,
|
||||
2.6645,
|
||||
2.6726,
|
||||
2.6803,
|
||||
2.6812,
|
||||
2.6899,
|
||||
2.6916,
|
||||
2.6931,
|
||||
2.6998,
|
||||
2.7062,
|
||||
2.7262,
|
||||
2.7222,
|
||||
2.7158,
|
||||
2.7041,
|
||||
2.7485,
|
||||
2.7491,
|
||||
2.7451,
|
||||
2.7485,
|
||||
2.7233,
|
||||
2.7297,
|
||||
2.7233,
|
||||
2.7145,
|
||||
2.6958,
|
||||
2.6788,
|
||||
2.6439,
|
||||
2.6007,
|
||||
2.4786,
|
||||
2.2469,
|
||||
2.1877,
|
||||
2.1392,
|
||||
2.0717,
|
||||
2.0107,
|
||||
1.9676,
|
||||
1.9140,
|
||||
1.7102,
|
||||
0.9101,
|
||||
0.7164,
|
||||
]
|
||||
|
||||
|
||||
_NORMALIZATION_EPSILON = 1e-4
|
||||
_SILU_DIVISOR = 0.596
|
||||
_RESIDUAL_BLEND = 0.3
|
||||
@@ -516,33 +77,22 @@ class ResnetBlock1D(nn.Module):
|
||||
*,
|
||||
in_dim: int,
|
||||
out_dim: int | None = None,
|
||||
conv_shortcut: bool = False,
|
||||
kernel_size: int = 3,
|
||||
use_norm: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.in_dim = in_dim
|
||||
self.out_dim = in_dim if out_dim is None else out_dim
|
||||
self.use_conv_shortcut = conv_shortcut
|
||||
self.use_norm = use_norm
|
||||
self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size)
|
||||
self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size)
|
||||
if self.in_dim != self.out_dim:
|
||||
if conv_shortcut:
|
||||
self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size)
|
||||
else:
|
||||
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
|
||||
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.use_norm:
|
||||
x = _rms_normalize(x, dim=1)
|
||||
x = _rms_normalize(x, dim=1)
|
||||
hidden = self.conv1(F.silu(x) / _SILU_DIVISOR)
|
||||
hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR)
|
||||
if self.in_dim != self.out_dim:
|
||||
shortcut = (
|
||||
self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut
|
||||
)
|
||||
x = shortcut(x)
|
||||
x = self.nin_shortcut(x)
|
||||
return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
|
||||
|
||||
|
||||
@@ -566,30 +116,60 @@ class AttnBlock1D(nn.Module):
|
||||
|
||||
|
||||
class Upsample1D(nn.Module):
|
||||
def __init__(self, in_channels: int, with_conv: bool) -> None:
|
||||
def __init__(self, in_channels: int) -> None:
|
||||
super().__init__()
|
||||
self.with_conv = with_conv
|
||||
if with_conv:
|
||||
self.conv = _conv1d(in_channels, in_channels, 3)
|
||||
self.conv = _conv1d(in_channels, in_channels, 3)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact")
|
||||
return self.conv(x) if self.with_conv else x
|
||||
return self.conv(x)
|
||||
|
||||
|
||||
class Downsample1D(nn.Module):
|
||||
def __init__(self, in_channels: int, with_conv: bool) -> None:
|
||||
def __init__(self, in_channels: int) -> None:
|
||||
super().__init__()
|
||||
self.with_conv = with_conv
|
||||
if with_conv:
|
||||
self.conv1 = _conv1d(in_channels, in_channels, 1)
|
||||
self.conv2 = _conv1d(in_channels, in_channels, 1)
|
||||
self.conv1 = _conv1d(in_channels, in_channels, 1)
|
||||
self.conv2 = _conv1d(in_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.with_conv:
|
||||
x = self.conv1(x)
|
||||
x = F.avg_pool1d(x, kernel_size=2, stride=2)
|
||||
return self.conv2(x) if self.with_conv else x
|
||||
return self.conv2(F.avg_pool1d(self.conv1(x), kernel_size=2, stride=2))
|
||||
|
||||
|
||||
def _mid_block(channels: int, kernel_size: int = 3) -> nn.Sequential:
|
||||
return nn.Sequential(
|
||||
OrderedDict(
|
||||
block_1=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size),
|
||||
attn_1=AttnBlock1D(channels),
|
||||
block_2=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class ResolutionBlock1D(nn.Module):
|
||||
"""Residual/attention stack with checkpoint-compatible block and attn names."""
|
||||
|
||||
def __init__(self, in_dim, out_dim, count, attention, kernel_size=3):
|
||||
super().__init__()
|
||||
self.block = nn.ModuleList()
|
||||
self.attn = nn.ModuleList()
|
||||
for index in range(count):
|
||||
self.block.append(
|
||||
ResnetBlock1D(
|
||||
in_dim=in_dim if index == 0 else out_dim,
|
||||
out_dim=out_dim,
|
||||
kernel_size=kernel_size,
|
||||
)
|
||||
)
|
||||
if attention:
|
||||
self.attn.append(AttnBlock1D(out_dim))
|
||||
|
||||
def forward(self, x, clip_act):
|
||||
for index, block in enumerate(self.block):
|
||||
x = block(x)
|
||||
if self.attn:
|
||||
x = self.attn[index](x)
|
||||
x = x.clamp(-clip_act, clip_act)
|
||||
return x
|
||||
|
||||
|
||||
class Encoder1D(nn.Module):
|
||||
@@ -603,73 +183,38 @@ class Encoder1D(nn.Module):
|
||||
down_layers: list[int],
|
||||
in_dim: int,
|
||||
embed_dim: int,
|
||||
resamp_with_conv: bool = True,
|
||||
double_z: bool = True,
|
||||
kernel_size: int = 3,
|
||||
clip_act: float = 256.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.num_layers = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.in_channels = in_dim
|
||||
self.clip_act = clip_act
|
||||
self.down_layers = down_layers
|
||||
self.attn_layers = attn_layers
|
||||
self.conv_in = _conv1d(in_dim, dim, kernel_size)
|
||||
|
||||
in_ch_mult = (1,) + ch_mult
|
||||
self.in_ch_mult = in_ch_mult
|
||||
self.down = nn.ModuleList()
|
||||
for level in range(self.num_layers):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_in = dim * in_ch_mult[level]
|
||||
block_out = dim * ch_mult[level]
|
||||
for _ in range(num_res_blocks):
|
||||
block.append(
|
||||
ResnetBlock1D(
|
||||
in_dim=block_in,
|
||||
out_dim=block_out,
|
||||
kernel_size=kernel_size,
|
||||
use_norm=True,
|
||||
)
|
||||
)
|
||||
block_in = block_out
|
||||
if level in attn_layers:
|
||||
attn.append(AttnBlock1D(block_in))
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
down.attn = attn
|
||||
down = ResolutionBlock1D(
|
||||
block_in, block_out, num_res_blocks, level in attn_layers, kernel_size
|
||||
)
|
||||
if level in down_layers:
|
||||
down.downsample = Downsample1D(block_in, resamp_with_conv)
|
||||
down.downsample = Downsample1D(block_out)
|
||||
self.down.append(down)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
|
||||
)
|
||||
self.mid.attn_1 = AttnBlock1D(block_in)
|
||||
self.mid.block_2 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
|
||||
)
|
||||
output_dim = 2 * embed_dim if double_z else embed_dim
|
||||
self.conv_out = _conv1d(block_in, output_dim, kernel_size)
|
||||
self.mid = _mid_block(block_out, kernel_size)
|
||||
self.conv_out = _conv1d(block_out, 2 * embed_dim, kernel_size)
|
||||
self.learnable_gain = nn.Parameter(torch.zeros([]))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
states = [self.conv_in(x)]
|
||||
for level in range(self.num_layers):
|
||||
for block_index in range(self.num_res_blocks):
|
||||
hidden = self.down[level].block[block_index](states[-1])
|
||||
if len(self.down[level].attn) > 0:
|
||||
hidden = self.down[level].attn[block_index](hidden)
|
||||
states.append(hidden.clamp(-self.clip_act, self.clip_act))
|
||||
hidden = self.conv_in(x)
|
||||
for level, down in enumerate(self.down):
|
||||
hidden = down(hidden, self.clip_act)
|
||||
if level in self.down_layers:
|
||||
states.append(self.down[level].downsample(states[-1]))
|
||||
hidden = self.mid.block_1(states[-1])
|
||||
hidden = self.mid.attn_1(hidden)
|
||||
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
hidden = down.downsample(hidden)
|
||||
hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
return _conv1d_with_gain(
|
||||
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
|
||||
)
|
||||
@@ -685,47 +230,27 @@ class Decoder1D(nn.Module):
|
||||
num_res_blocks: int,
|
||||
attn_layers: list[int],
|
||||
down_layers: list[int],
|
||||
in_dim: int,
|
||||
embed_dim: int,
|
||||
kernel_size: int = 3,
|
||||
resamp_with_conv: bool = True,
|
||||
clip_act: float = 256.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.ch = dim
|
||||
self.num_layers = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.in_channels = in_dim
|
||||
self.clip_act = clip_act
|
||||
self.down_layers = [level + 1 for level in down_layers]
|
||||
block_in = dim * ch_mult[-1]
|
||||
self.conv_in = _conv1d(embed_dim, block_in, kernel_size)
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, use_norm=True
|
||||
)
|
||||
self.mid.attn_1 = AttnBlock1D(block_in)
|
||||
self.mid.block_2 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, use_norm=True
|
||||
)
|
||||
self.mid = _mid_block(block_in)
|
||||
|
||||
self.up = nn.ModuleList()
|
||||
for level in reversed(range(self.num_layers)):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_out = dim * ch_mult[level]
|
||||
for _ in range(num_res_blocks + 1):
|
||||
block.append(
|
||||
ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True)
|
||||
)
|
||||
block_in = block_out
|
||||
if level in attn_layers:
|
||||
attn.append(AttnBlock1D(block_in))
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
up.attn = attn
|
||||
up = ResolutionBlock1D(
|
||||
block_in, block_out, num_res_blocks + 1, level in attn_layers
|
||||
)
|
||||
block_in = block_out
|
||||
if level in self.down_layers:
|
||||
up.upsample = Upsample1D(block_in, resamp_with_conv)
|
||||
up.upsample = Upsample1D(block_in)
|
||||
self.up.insert(0, up)
|
||||
|
||||
self.conv_out = _conv1d(block_in, out_dim, kernel_size)
|
||||
@@ -733,15 +258,9 @@ class Decoder1D(nn.Module):
|
||||
|
||||
def forward(self, z: torch.Tensor) -> torch.Tensor:
|
||||
hidden = self.conv_in(z)
|
||||
hidden = self.mid.block_1(hidden)
|
||||
hidden = self.mid.attn_1(hidden)
|
||||
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
for level in reversed(range(self.num_layers)):
|
||||
for block_index in range(self.num_res_blocks + 1):
|
||||
hidden = self.up[level].block[block_index](hidden)
|
||||
if len(self.up[level].attn) > 0:
|
||||
hidden = self.up[level].attn[block_index](hidden)
|
||||
hidden = hidden.clamp(-self.clip_act, self.clip_act)
|
||||
hidden = self.up[level](hidden, self.clip_act)
|
||||
if level in self.down_layers:
|
||||
hidden = self.up[level].upsample(hidden)
|
||||
return _conv1d_with_gain(
|
||||
@@ -752,32 +271,26 @@ class Decoder1D(nn.Module):
|
||||
class MMAudioVAE(nn.Module):
|
||||
"""MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio."""
|
||||
|
||||
def __init__(
|
||||
self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False
|
||||
) -> None:
|
||||
def __init__(self, mode: str = "44k", need_encoder: bool = False) -> None:
|
||||
super().__init__()
|
||||
if isinstance(mode, dict):
|
||||
config = mode
|
||||
mode = config.get("mode", "44k")
|
||||
need_encoder = config.get("need_encoder", need_encoder)
|
||||
if mode == "16k":
|
||||
data_dim, embed_dim, hidden_dim = 80, 20, 384
|
||||
data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D
|
||||
elif mode == "44k":
|
||||
data_dim, embed_dim, hidden_dim = 128, 40, 512
|
||||
data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D
|
||||
else:
|
||||
raise ValueError(f"Unknown MMAudio VAE mode: {mode}")
|
||||
|
||||
self.mode = mode
|
||||
self.embed_dim = embed_dim
|
||||
self._weights_normalized = False
|
||||
# checkpoint buffers are required by native VAELoader's strict load
|
||||
self.register_buffer(
|
||||
"data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1)
|
||||
"data_mean", torch.zeros(1, data_dim, 1, dtype=torch.float32)
|
||||
)
|
||||
self.register_buffer(
|
||||
"data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1)
|
||||
"data_std", torch.ones(1, data_dim, 1, dtype=torch.float32)
|
||||
)
|
||||
self.encoder = None
|
||||
if need_encoder:
|
||||
self.encoder = Encoder1D(
|
||||
dim=hidden_dim,
|
||||
@@ -794,7 +307,6 @@ class MMAudioVAE(nn.Module):
|
||||
num_res_blocks=2,
|
||||
attn_layers=[3],
|
||||
down_layers=[0],
|
||||
in_dim=data_dim,
|
||||
out_dim=data_dim,
|
||||
embed_dim=embed_dim,
|
||||
)
|
||||
@@ -803,7 +315,7 @@ class MMAudioVAE(nn.Module):
|
||||
self, mel: torch.Tensor, normalize_input: bool = True
|
||||
) -> DiagonalGaussianDistribution:
|
||||
self._require_normalized_weights()
|
||||
if not hasattr(self, "encoder"):
|
||||
if self.encoder is None:
|
||||
raise RuntimeError("This MMAudio VAE was loaded decoder-only")
|
||||
if normalize_input:
|
||||
mel = self.normalize(mel)
|
||||
@@ -831,17 +343,7 @@ class MMAudioVAE(nn.Module):
|
||||
|
||||
@torch.no_grad()
|
||||
def remove_weight_norm(self) -> MMAudioVAE:
|
||||
# Idempotent: the diffusers reference calls this exactly once, in
|
||||
# ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the
|
||||
# freshly-constructed (random) Conv1d weights with a checkpoint's
|
||||
# real, already-final values -- so a real checkpoint's own weights
|
||||
# are never renormalized. Mirroring that timing here (see
|
||||
# ``Kandinsky6AudioVAE.__init__``) means this method only ever runs
|
||||
# against pre-load weights too; the guard below additionally makes a
|
||||
# second, redundant call (e.g. the wrapper's own public
|
||||
# ``remove_weight_norm()``) a true no-op instead of re-mutating an
|
||||
# already-normalized (possibly checkpoint-loaded) weight a second
|
||||
# time.
|
||||
# normalize only random initialization; checkpoint weights are already final
|
||||
if self._weights_normalized:
|
||||
return self
|
||||
for name, module in self.named_modules():
|
||||
|
||||
@@ -38,15 +38,19 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm
|
||||
|
||||
# Pixel frames per encode segment after the first (which has one more: the causal first frame).
|
||||
# pixel frames per segment after the extra causal first frame
|
||||
_SEGMENT_FRAMES = 16
|
||||
# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles
|
||||
# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9
|
||||
# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv
|
||||
# workspace to a fraction of the activation.
|
||||
# temporal chunks bound workspace and avoid int32 indexing overflow at 4x decode
|
||||
_MAX_CONV_NUMEL = 2 * 10**9
|
||||
|
||||
|
||||
def _segment_sizes(frames: int, stride: int) -> list[int]:
|
||||
"""The first causal segment has one extra frame; the final segment may be short."""
|
||||
return [min(frames, stride + 1)] + [
|
||||
min(stride, frames - start) for start in range(stride + 1, frames, stride)
|
||||
]
|
||||
|
||||
|
||||
class _SegmentCache:
|
||||
"""Causal state carried from one temporal segment to the next during a single encode / decode call."""
|
||||
|
||||
@@ -56,17 +60,12 @@ class _SegmentCache:
|
||||
|
||||
|
||||
def _silu(x: torch.Tensor) -> torch.Tensor:
|
||||
# The output norms use this form and the resblocks use F.silu; the two are not bitwise
|
||||
# equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is
|
||||
# deliberate, not a simplification opportunity.
|
||||
# output norms require this form; F.silu in resblocks rounds differently in bf16
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
class _K6RMSNorm(WanRMS_norm):
|
||||
"""``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the
|
||||
Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for
|
||||
fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably
|
||||
drifts from the reference over many resnet blocks."""
|
||||
"""Wan RMS norm with the reference's unconditional fp32 reduction."""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
normed = F.normalize(x.float(), dim=1).to(x.dtype)
|
||||
@@ -89,10 +88,7 @@ class _ChunkedConv3d(nn.Conv3d):
|
||||
f"padding {self.padding}"
|
||||
)
|
||||
if self.stride[0] != 1:
|
||||
# A strided conv can't reuse the stride-1 path's "prefix each chunk with the
|
||||
# previous chunk's tail" trick below (a strided conv's output doesn't align with
|
||||
# input chunk boundaries), so slide one kernel-width window at a time instead, only
|
||||
# raising if even a single window would overflow.
|
||||
# strided outputs may cross chunk boundaries: use one kernel window per output
|
||||
stride = self.stride[0]
|
||||
window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4]
|
||||
if window_numel >= _MAX_CONV_NUMEL:
|
||||
@@ -100,8 +96,6 @@ class _ChunkedConv3d(nn.Conv3d):
|
||||
f"frames are too big for Conv3d even one stride-{stride} window at a time "
|
||||
f"({window_numel} elements per window)"
|
||||
)
|
||||
# Bare super() doesn't resolve inside a list comprehension's own implicit scope,
|
||||
# hence the loop.
|
||||
window_outputs = []
|
||||
for i in range(0, frames - k + 1, stride):
|
||||
window_outputs.append(super().forward(x[:, :, i : i + k]))
|
||||
@@ -229,21 +223,14 @@ class _ResnetBlock3D(nn.Module):
|
||||
if in_channels != out_channels:
|
||||
self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1)
|
||||
|
||||
def _norm(
|
||||
self,
|
||||
norm: nn.Module,
|
||||
h: torch.Tensor,
|
||||
zq: torch.Tensor | None,
|
||||
cache: _SegmentCache,
|
||||
) -> torch.Tensor:
|
||||
return norm(h) if zq is None else norm(h, zq, cache)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True)
|
||||
h = self.norm1(x) if zq is None else self.norm1(x, zq, cache)
|
||||
h = F.silu(h, inplace=True)
|
||||
h = self.conv1(h, cache)
|
||||
h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True)
|
||||
h = self.norm2(h) if zq is None else self.norm2(h, zq, cache)
|
||||
h = F.silu(h, inplace=True)
|
||||
h = self.conv2(h, cache)
|
||||
if hasattr(self, "nin_shortcut"):
|
||||
x = self.nin_shortcut(x)
|
||||
@@ -375,8 +362,6 @@ class _PXSUpsample(nn.Module):
|
||||
x = x[:, :, 1:]
|
||||
x = self.temporal_conv(x, cache) + x
|
||||
frames, height, width = x.shape[-3:]
|
||||
# Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow
|
||||
# this guards against (the old vendored code did not chunk this specific call).
|
||||
x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2))
|
||||
x.add_(self.spatial_conv(x))
|
||||
return self.linear(x)
|
||||
@@ -402,7 +387,6 @@ class _Encoder3D(nn.Module):
|
||||
temporal_compress_start_level: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_res_blocks = num_res_blocks
|
||||
time_levels = range(
|
||||
temporal_compress_start_level,
|
||||
temporal_compress_start_level + int(math.log2(temporal_compress_times)),
|
||||
@@ -494,10 +478,7 @@ class _Decoder3D(nn.Module):
|
||||
return self.conv_out(_silu(self.norm_out(h, z, cache)), cache)
|
||||
|
||||
|
||||
# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything
|
||||
# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``,
|
||||
# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored
|
||||
# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params.
|
||||
# constructor fields; resolution and training-only metadata do not affect the architecture
|
||||
_ARCH_KEYS = (
|
||||
"ch",
|
||||
"ch_mult",
|
||||
@@ -508,12 +489,7 @@ _ARCH_KEYS = (
|
||||
|
||||
|
||||
def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]:
|
||||
# norm_type is the one legacy knob this port still checks: unlike padding_mode /
|
||||
# downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer
|
||||
# branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride /
|
||||
# double-z behavior those knobs used to select), a group_norm checkpoint would silently
|
||||
# get rms_norm layers instead of a clear error, which is the one mismatch worth failing
|
||||
# loudly on here.
|
||||
# never silently substitute RMSNorm for an unsupported group-norm checkpoint
|
||||
norm_type = conf.get("norm_type", "rms_norm")
|
||||
if norm_type != "rms_norm":
|
||||
raise ValueError(
|
||||
@@ -596,12 +572,7 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
|
||||
"""``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``."""
|
||||
split_list = [_SEGMENT_FRAMES + 1]
|
||||
remaining = x.size(2) - split_list[0]
|
||||
while remaining > 0:
|
||||
split_list.append(_SEGMENT_FRAMES)
|
||||
remaining -= _SEGMENT_FRAMES
|
||||
split_list[-1] += remaining
|
||||
split_list = _segment_sizes(x.size(2), _SEGMENT_FRAMES)
|
||||
cache = _SegmentCache()
|
||||
latents = []
|
||||
for segment in torch.split(x, split_list, dim=2):
|
||||
@@ -634,15 +605,9 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
return DecoderOutput(sample=self._decode(z))
|
||||
|
||||
def _decode(self, z: torch.Tensor) -> torch.Tensor:
|
||||
segment = _SEGMENT_FRAMES // self.temporal_compression
|
||||
num_frames = z.size(2)
|
||||
if num_frames == 1:
|
||||
split_list = [1]
|
||||
else:
|
||||
split_list = [segment] * ((num_frames - 1) // segment)
|
||||
if (num_frames - 1) % segment:
|
||||
split_list.append((num_frames - 1) % segment)
|
||||
split_list[0] += 1
|
||||
split_list = _segment_sizes(
|
||||
z.size(2), _SEGMENT_FRAMES // self.temporal_compression
|
||||
)
|
||||
cache = _SegmentCache()
|
||||
samples = []
|
||||
for chunk in torch.split(z, split_list, dim=2):
|
||||
|
||||
@@ -15,22 +15,12 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
|
||||
Kandinsky6SREncodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
|
||||
Kandinsky6SROutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
+11
-57
@@ -29,6 +29,9 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
|
||||
@@ -48,17 +51,8 @@ from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYP
|
||||
class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
"""Run the Kandinsky6 joint video+audio denoising loop."""
|
||||
|
||||
def __init__(self, transformer, scheduler, pipeline=None) -> None:
|
||||
super().__init__(
|
||||
transformer=transformer, scheduler=scheduler, pipeline=pipeline
|
||||
)
|
||||
|
||||
def _owns_compile_warmup_lifecycle(self) -> bool:
|
||||
# forward() overrides DenoisingStage.forward (the joint video+audio
|
||||
# loop cannot reuse the parent's single-tensor _denoise()), but it
|
||||
# still wraps the loop in `_offload_for_torch_compile_warmup` itself
|
||||
# below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's
|
||||
# other joint-modality denoising stage.
|
||||
# forward wraps the joint loop in the shared compile warmup lifecycle
|
||||
return True
|
||||
|
||||
def component_uses(
|
||||
@@ -155,7 +149,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
t_expand: torch.Tensor,
|
||||
visual_rope_pos: list[torch.Tensor],
|
||||
scale_factor: tuple[float, ...],
|
||||
sparse_params: dict[str, Any] | None,
|
||||
visual_token_type_ids: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run sequential or distributed CFG branches and combine video/audio velocity."""
|
||||
@@ -172,7 +165,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
timestep=t_expand,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
sparse_params=sparse_params,
|
||||
visual_token_type_ids=visual_token_type_ids,
|
||||
# BCG clones tuple leaves so the next CFG replay cannot
|
||||
# overwrite the previous branch's predictions
|
||||
@@ -227,7 +219,7 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
scheduler = batch.scheduler
|
||||
if scheduler is None:
|
||||
raise ValueError("scheduler must be set for Kandinsky6 denoising.")
|
||||
use_piflow = bool(getattr(scheduler, "is_piflow", False))
|
||||
use_piflow = isinstance(scheduler, PiflowScheduler)
|
||||
if use_piflow and (
|
||||
not math.isfinite(batch.guidance_scale)
|
||||
or abs(batch.guidance_scale - 1.0) > 1e-6
|
||||
@@ -237,30 +229,10 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
|
||||
pipeline_config = server_args.pipeline_config
|
||||
arch = pipeline_config.dit_config.arch_config
|
||||
# The runtime transformer already rejects attention_engine="nabla"
|
||||
# at construction time (NABLA sparse attention is not yet ported),
|
||||
# so sparse_params stays None on every currently-reachable
|
||||
# configuration. A future NABLA-capable DiT would fill this in.
|
||||
if arch.attention_engine == "nabla":
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6 NABLA sparse-attention metadata construction is not implemented; "
|
||||
"the runtime transformer also rejects attention_engine='nabla' at construction "
|
||||
"time. Use attention_engine='auto' or 'sdpa'."
|
||||
)
|
||||
sparse_params = None
|
||||
|
||||
device = get_local_torch_device()
|
||||
target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
|
||||
|
||||
# .clone(): batch.latents was created by Kandinsky6LatentPreparationStage
|
||||
# under an active torch.inference_mode() context further up the
|
||||
# pipeline, which marks it as an "inference tensor" -- such tensors
|
||||
# can only be mutated in place while still inside an inference_mode
|
||||
# context. This stage runs under plain @torch.no_grad() instead, and
|
||||
# mutates `video` in place every denoising step (video[..., :num_channels]
|
||||
# = ...), so clone once here to get an ordinary, freely-mutable tensor
|
||||
# before the loop starts. audio is never mutated in place (only
|
||||
# reassigned via out-of-place `+`), so it needs no clone.
|
||||
# clone inference tensors before the in-place video updates under no_grad
|
||||
video = batch.latents.clone()
|
||||
audio = batch.audio_latents
|
||||
num_channels = int(arch.in_visual_dim)
|
||||
@@ -295,40 +267,25 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
|
||||
patch_size = arch.patch_size
|
||||
|
||||
# Constant per-request geometry -- computed once outside the loop,
|
||||
# not per-step.
|
||||
# geometry is constant across steps
|
||||
num_video_frames = video.shape[1] - (1 if tail_cond else 0)
|
||||
t_positions = torch.arange(num_video_frames, device=device)
|
||||
if tail_cond:
|
||||
# The appended reference frame reuses T-position 0's rope row
|
||||
# (RoPE3D is a pure position -> table lookup, so duplicating the
|
||||
# first row is equivalent to the diffusers reference's
|
||||
# ``torch.cat([rope, rope[:1]])``).
|
||||
# the reference frame reuses time position zero
|
||||
t_positions = torch.cat([t_positions, t_positions.new_zeros(1)])
|
||||
visual_rope_pos = [
|
||||
t_positions,
|
||||
torch.arange(height // spatial_ratio // patch_size[1], device=device),
|
||||
torch.arange(width // spatial_ratio // patch_size[2], device=device),
|
||||
]
|
||||
# Fixed per-checkpoint RoPE frequency scaling read from the DiT's
|
||||
# arch config (transformer/config.json's "scale_factor", both real
|
||||
# Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers
|
||||
# reference, which resolves this once in
|
||||
# ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every
|
||||
# request regardless of the request's own height/width. NOT a
|
||||
# function of the request's resolution.
|
||||
# RoPE scaling comes from the checkpoint, not the request resolution
|
||||
scale_factor = arch.scale_factor
|
||||
image_latent = batch.image_latent
|
||||
|
||||
total_steps = int(batch.timesteps.shape[0])
|
||||
self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch)
|
||||
|
||||
# PiFlow's noisy video state is tracked in fp32 across steps here,
|
||||
# separately from the `video` buffer's bf16 storage dtype -- see the
|
||||
# module docstring. Always a Tensor (not `Tensor | None`) even on the
|
||||
# flow-Euler path, where it is simply unused: the initial cast is one
|
||||
# cheap op, and keeping it non-Optional avoids re-narrowing it after
|
||||
# every `if use_piflow:` branch re-entry in the loop below.
|
||||
# pi-Flow retains fp32 state between bf16 DiT calls
|
||||
video_state = video[..., :num_channels].to(torch.float32)
|
||||
|
||||
with self.use_declared_component(
|
||||
@@ -371,7 +328,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
t_expand=t_expand,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
sparse_params=sparse_params,
|
||||
visual_token_type_ids=visual_token_type_ids,
|
||||
)
|
||||
|
||||
@@ -397,9 +353,7 @@ class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
if use_piflow:
|
||||
video_state[:, -1:] = ref_frame.to(torch.float32)
|
||||
|
||||
# Manual Euler update for audio, using the SAME per-step
|
||||
# sigma delta the scheduler.step() call above just
|
||||
# consumed for video.
|
||||
# Euler audio reuses the video's sigma delta without advancing it
|
||||
if use_piflow:
|
||||
audio = audio_scheduler.step(
|
||||
audio_vel, timestep, audio, return_dict=False
|
||||
|
||||
+1
-7
@@ -1,8 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers.
|
||||
|
||||
Import the stage classes from their modules (``input_stage``, ``encode_stage``,
|
||||
``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package
|
||||
``__init__`` stays empty so that configs can import the pure helpers (``video_utils``,
|
||||
``tiling``, ...) cheaply.
|
||||
"""
|
||||
"""SR stages and helpers; import stages explicitly to keep config imports lightweight."""
|
||||
|
||||
-78
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_local_torch_device,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
has_decode_parallel_world,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_DENOISED_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
decode_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
|
||||
|
||||
|
||||
class Kandinsky6SRDecodeStage(PipelineStage):
|
||||
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
|
||||
|
||||
def __init__(self, vae) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="decode_tiles",
|
||||
target_dtype=vae_dtype,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
denoised = batch.extra.pop(SR_DENOISED_KEY)
|
||||
device = get_local_torch_device()
|
||||
group = None
|
||||
if (
|
||||
server_args.pipeline_config.vae_config.use_parallel_tiling
|
||||
and has_decode_parallel_world()
|
||||
and not can_install_spatial_shard_parallel_decode(
|
||||
server_args.pipeline_config.vae_config
|
||||
)
|
||||
):
|
||||
group = get_decode_parallel_group_coordinator()
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="decode_tiles"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
tiles = decode_chunks(
|
||||
denoised,
|
||||
vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
device=device,
|
||||
group=group,
|
||||
)
|
||||
batch.extra[SR_TILES_KEY] = tiles
|
||||
return batch
|
||||
-137
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks.
|
||||
|
||||
Each chunk resets its scheduler; the generic single-latent CFG loop and
|
||||
prompt-padding BCG path do not apply."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_CHUNKS_KEY,
|
||||
SR_DENOISED_KEY,
|
||||
SR_DIT_SPEC_KEY,
|
||||
SR_SAMPLING_SPEC_KEY,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
denoise_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype
|
||||
|
||||
|
||||
class Kandinsky6SRDenoisingStage(DenoisingStage):
|
||||
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
|
||||
|
||||
def __init__(self, transformer, scheduler, pipeline=None) -> None:
|
||||
super().__init__(
|
||||
transformer=transformer,
|
||||
scheduler=scheduler,
|
||||
pipeline=pipeline,
|
||||
transformer_2=None,
|
||||
vae=None,
|
||||
)
|
||||
|
||||
def _owns_compile_warmup_lifecycle(self) -> bool:
|
||||
# ``forward`` below does not go through the shared ``_denoise`` loop that this guard
|
||||
# normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage):
|
||||
# the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``.
|
||||
return True
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_SAMPLING_SPEC_KEY]",
|
||||
batch.extra.get(SR_SAMPLING_SPEC_KEY),
|
||||
V.not_none,
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
dit_dtype = precision_to_dtype(
|
||||
server_args.pipeline_config.dit_precision, "dit_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"transformer",
|
||||
phase="denoise_tiles",
|
||||
target_dtype=dit_dtype,
|
||||
preferred_ready_after_request=True,
|
||||
memory_intensive=True,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
with self._offload_for_torch_compile_warmup(batch):
|
||||
return self._denoise_tiles(batch, server_args)
|
||||
|
||||
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
|
||||
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
|
||||
chunks = batch.extra.pop(SR_CHUNKS_KEY)
|
||||
device = get_local_torch_device()
|
||||
|
||||
# Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the
|
||||
# corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this
|
||||
# request's DiT-call count, the closest analogue of the global ``num_inference_steps``
|
||||
# those hooks expect.
|
||||
self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch)
|
||||
|
||||
scheduler = effective_scheduler(spec, self.scheduler)
|
||||
with self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer, phase="denoise_tiles"
|
||||
) as transformer:
|
||||
assert transformer is not None
|
||||
self.transformer = transformer
|
||||
total = spec.steps_per_chunk * len(chunks)
|
||||
with self.progress_bar(
|
||||
total=total, batch=batch, desc="Kandinsky6 SR denoising"
|
||||
) as progress:
|
||||
denoised = denoise_chunks(
|
||||
chunks,
|
||||
transformer,
|
||||
scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
step_context=lambda step: set_forward_context(
|
||||
current_timestep=step, attn_metadata=None, forward_batch=batch
|
||||
),
|
||||
on_step=progress.update,
|
||||
)
|
||||
self._finish_active_component_use()
|
||||
batch.extra[SR_DENOISED_KEY] = denoised
|
||||
return batch
|
||||
-65
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
uses_latent_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
encode_video_to_lr_latent,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
|
||||
|
||||
|
||||
class Kandinsky6SREncodeStage(PipelineStage):
|
||||
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
|
||||
|
||||
def __init__(self, vae, latent_upscaler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
# Only consulted to pick the path (does the bank have this scale?).
|
||||
self.latent_upscaler = latent_upscaler
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="encode_video",
|
||||
target_dtype=vae_dtype,
|
||||
start_at_stage_entry=False,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
if not uses_latent_path(self.latent_upscaler, tiling_scale):
|
||||
return batch
|
||||
video = batch.extra.pop(SR_VIDEO_KEY)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="encode_video"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
lr_latent = encode_video_to_lr_latent(
|
||||
video, vae, device=get_local_torch_device()
|
||||
)
|
||||
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
|
||||
return batch
|
||||
-95
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio)."""
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
VAE_SPATIAL_FACTOR,
|
||||
pad_to_spatial_factor,
|
||||
pre_upscale_video,
|
||||
resolve_scale_request,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
SOURCE_AUDIO_SAMPLE_RATE,
|
||||
DecodedClip,
|
||||
decode_clip,
|
||||
extract_source_audio,
|
||||
synthetic_clip,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
# Warmup requests carry no (usable) video: run the real path on a small synthetic clip.
|
||||
WARMUP_CLIP_FRAMES = 9
|
||||
WARMUP_CLIP_HW = (256, 384)
|
||||
|
||||
|
||||
class Kandinsky6SRInputStage(PipelineStage):
|
||||
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
|
||||
|
||||
Record requested dimensions before VAE padding so output can remove it."""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
|
||||
clip = self._load_clip(batch)
|
||||
video = clip.frames
|
||||
if pre_upscale != 1.0:
|
||||
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
|
||||
frames = video.shape[0]
|
||||
# Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel
|
||||
# pre-upscale happens to already land on an aligned size): without this, a source
|
||||
# whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4,
|
||||
# which runs no pixel resize at all) fails partway through the whole-video KVAE
|
||||
# encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps
|
||||
# every source pixel; batch.height / batch.width below, and the output stage's final
|
||||
# crop, restore the originally requested dimensions.
|
||||
video, (requested_h, requested_w) = pad_to_spatial_factor(
|
||||
video, VAE_SPATIAL_FACTOR
|
||||
)
|
||||
|
||||
batch.extra[SR_VIDEO_KEY] = video
|
||||
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
|
||||
batch.fps = clip.fps
|
||||
batch.num_frames = frames
|
||||
batch.height = requested_h * tiling_scale
|
||||
batch.width = requested_w * tiling_scale
|
||||
self._attach_source_audio(batch, frames=frames, fps=clip.fps)
|
||||
return batch
|
||||
|
||||
def _load_clip(self, batch: Req) -> DecodedClip:
|
||||
if batch.is_warmup:
|
||||
height, width = WARMUP_CLIP_HW
|
||||
frames = synthetic_clip(
|
||||
height=height, width=width, frames=WARMUP_CLIP_FRAMES
|
||||
)
|
||||
return DecodedClip(frames=frames, fps=24, source_fps=24.0)
|
||||
path = batch.sampling_params.source_video_path()
|
||||
if path is None or not os.path.isfile(path):
|
||||
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
|
||||
self.log_info("Decoding %s", path)
|
||||
return decode_clip(path)
|
||||
|
||||
@staticmethod
|
||||
def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None:
|
||||
if batch.is_warmup:
|
||||
return
|
||||
path = batch.sampling_params.source_video_path()
|
||||
audio = extract_source_audio(
|
||||
path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps
|
||||
)
|
||||
if audio is None:
|
||||
batch.audio = None
|
||||
batch.audio_sample_rate = None
|
||||
return
|
||||
batch.audio = torch.from_numpy(audio)
|
||||
batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE
|
||||
-162
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback.
|
||||
|
||||
Validate sampler/head compatibility and step count before processing tiles."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_CHUNKS_KEY,
|
||||
SR_DIT_SPEC_KEY,
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_PLAN_KEY,
|
||||
SR_SAMPLING_SPEC_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
check_denoising_request,
|
||||
uses_latent_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
TilePlan,
|
||||
plan_tiles,
|
||||
prepare_lu_tile_latents,
|
||||
prepare_pixel_tile_latents,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
VAE_SPATIAL_FACTOR,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
resolve_precision,
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentPrepStage(PipelineStage):
|
||||
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
|
||||
|
||||
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
self.transformer = transformer # arch-config / dtype only: never called here
|
||||
self.latent_upscaler = latent_upscaler
|
||||
self.scheduler = (
|
||||
scheduler # resolved sampler validation only: never stepped here
|
||||
)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
uses: list[ComponentUse] = []
|
||||
if self.latent_upscaler is not None:
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name,
|
||||
"latent_upscaler",
|
||||
phase="upscale_tiles",
|
||||
target_dtype=torch.bfloat16,
|
||||
)
|
||||
)
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
|
||||
)
|
||||
)
|
||||
return uses
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
arch = server_args.pipeline_config.dit_config.arch_config
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=tiling_scale,
|
||||
seed=seed,
|
||||
num_steps=batch.num_inference_steps,
|
||||
tiles_batch_size=batch.sr_tiles_batch_size,
|
||||
tile_min_overlap=batch.sr_tile_min_overlap,
|
||||
scheduler=self.scheduler,
|
||||
)
|
||||
check_denoising_request(spec, batch.num_inference_steps)
|
||||
dit_spec = build_dit_spec(self.transformer)
|
||||
device = get_local_torch_device()
|
||||
|
||||
plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device)
|
||||
batch.extra[SR_PLAN_KEY] = plan
|
||||
batch.extra[SR_CHUNKS_KEY] = chunks
|
||||
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
|
||||
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
|
||||
return batch
|
||||
|
||||
def _prepare_chunks(
|
||||
self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device
|
||||
) -> tuple[TilePlan, list[torch.Tensor]]:
|
||||
common = dict(
|
||||
visual_size=spec.visual_size,
|
||||
tiling_scale=spec.tiling_scale,
|
||||
tile_min_overlap=spec.tile_min_overlap,
|
||||
)
|
||||
if uses_latent_path(self.latent_upscaler, spec.tiling_scale):
|
||||
lr_latent = batch.extra.pop(SR_LR_LATENT_KEY)
|
||||
plan = plan_tiles(
|
||||
frame_hw=(
|
||||
lr_latent.shape[-2] * VAE_SPATIAL_FACTOR,
|
||||
lr_latent.shape[-1] * VAE_SPATIAL_FACTOR,
|
||||
),
|
||||
**common,
|
||||
)
|
||||
with self.use_declared_component(
|
||||
component_name="latent_upscaler",
|
||||
module=self.latent_upscaler,
|
||||
phase="upscale_tiles",
|
||||
) as bank:
|
||||
assert bank is not None
|
||||
self.latent_upscaler = bank
|
||||
chunks = prepare_lu_tile_latents(
|
||||
lr_latent,
|
||||
plan,
|
||||
upscale_fn=partial(bank.upscale, scale=spec.tiling_scale),
|
||||
lu_dtype=module_dtype(bank),
|
||||
scaling_factor=self.vae.scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
return plan, chunks
|
||||
|
||||
video = batch.extra.pop(SR_VIDEO_KEY)
|
||||
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="encode_tiles"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
chunks = prepare_pixel_tile_latents(
|
||||
video,
|
||||
plan,
|
||||
vae=vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
return plan, chunks
|
||||
+25
-79
@@ -35,23 +35,12 @@ def degrade_lq_latent(
|
||||
return (1 - noise_scale) * lq_latent + noise_scale * eps
|
||||
|
||||
|
||||
def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor:
|
||||
"""Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype)."""
|
||||
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
|
||||
return value.to(dtype=dtype)
|
||||
return value
|
||||
|
||||
|
||||
def build_initial_latent( # noqa: PLR0913
|
||||
def build_initial_latent(
|
||||
*,
|
||||
instruct_type: InstructType,
|
||||
visual_cond: bool,
|
||||
in_visual_dim: int,
|
||||
lq_latent: torch.Tensor,
|
||||
batch_size: int,
|
||||
duration: int,
|
||||
height: int,
|
||||
width: int,
|
||||
device: str | int | torch.device,
|
||||
seed: int,
|
||||
lq_noise_scale: float,
|
||||
@@ -59,76 +48,33 @@ def build_initial_latent( # noqa: PLR0913
|
||||
lq_channel_noise_scale: float,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build [starting | conditioning | mask] for the selected instruction mode.
|
||||
|
||||
Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except
|
||||
unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise."""
|
||||
lq_latent = lq_latent.to(device)
|
||||
lq_latent = _cast_floating(lq_latent, dtype)
|
||||
|
||||
"""Build packed [B*T, H, W, C] noise and optional conditioning without reseeding global RNG."""
|
||||
if instruct_type not in ("noise", "channel", "hybrid", "hybrid_anchor"):
|
||||
raise ValueError(f"Unsupported SR instruct_type={instruct_type!r}")
|
||||
lq_latent = lq_latent.to(device=device, dtype=dtype)
|
||||
generator = torch.Generator(device=device).manual_seed(seed)
|
||||
if instruct_type == "channel":
|
||||
starting = torch.randn(
|
||||
(*lq_latent.shape[:-1], in_visual_dim), device=device, generator=generator
|
||||
)
|
||||
else:
|
||||
starting = degrade_lq_latent(
|
||||
lq_latent, lq_noise_scale, lq_noise_type, generator=generator
|
||||
)
|
||||
if instruct_type == "noise":
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
degraded_lq = degrade_lq_latent(
|
||||
lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
if not visual_cond:
|
||||
# No conditioning channels in the model input layer (e.g. the
|
||||
# t2v-initialized KVAE SR DiT) — feed the bare C-channel latent.
|
||||
return degraded_lq
|
||||
zero_cond = torch.zeros_like(degraded_lq)
|
||||
zero_mask = torch.zeros(
|
||||
[*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device
|
||||
return starting
|
||||
conditioning = torch.zeros_like(starting)
|
||||
mask = torch.zeros_like(starting[..., :1])
|
||||
elif instruct_type == "hybrid_anchor":
|
||||
# no HR anchor exists at inference; conditioning noise still uses the global RNG
|
||||
conditioning = degrade_lq_latent(
|
||||
torch.zeros_like(lq_latent), lq_channel_noise_scale, noise_type="linear"
|
||||
)
|
||||
return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1)
|
||||
|
||||
if instruct_type in ("channel", "hybrid"):
|
||||
if instruct_type == "hybrid":
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
starting = degrade_lq_latent(
|
||||
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
else:
|
||||
g = torch.Generator(device=device)
|
||||
g.manual_seed(seed)
|
||||
starting = torch.randn(
|
||||
batch_size * duration,
|
||||
height,
|
||||
width,
|
||||
in_visual_dim,
|
||||
device=device,
|
||||
generator=g,
|
||||
)
|
||||
channel_lq = degrade_lq_latent(
|
||||
mask = torch.zeros_like(lq_latent[..., :1])
|
||||
else:
|
||||
conditioning = degrade_lq_latent(
|
||||
lq_latent, lq_channel_noise_scale, noise_type="linear"
|
||||
)
|
||||
mask = torch.ones_like(lq_latent[..., :1])
|
||||
return torch.cat([starting, channel_lq, mask], dim=-1)
|
||||
|
||||
if instruct_type == "hybrid_anchor":
|
||||
# Anchor-free inference (tiled real-world LQ has no HR ground truth):
|
||||
# a zeroed anchor + zeroed mask runs the model with no anchor signal,
|
||||
# equivalent to anchor_indices=[] for the packed-latent datasets.
|
||||
anchor_latent = torch.zeros_like(lq_latent)
|
||||
anchor_mask = torch.zeros_like(lq_latent[..., :1])
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
starting = degrade_lq_latent(
|
||||
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
anchor = degrade_lq_latent(
|
||||
anchor_latent, lq_channel_noise_scale, noise_type="linear"
|
||||
)
|
||||
return torch.cat([starting, anchor, anchor_mask], dim=-1)
|
||||
|
||||
msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"InstructType",
|
||||
"LqNoiseType",
|
||||
"build_initial_latent",
|
||||
"degrade_lq_latent",
|
||||
]
|
||||
return torch.cat([starting, conditioning, mask], dim=-1)
|
||||
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Stitch / crop / resize / output stage of Kandinsky 6 video SR."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
resolve_target_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_PLAN_KEY,
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
stitch_tiles,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
crop_to_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
to_output_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
resize_to_target,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class Kandinsky6SROutputStage(PipelineStage):
|
||||
"""Hann-blends the decoded tiles, restores the requested output size, applies the
|
||||
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
|
||||
plus the mono source audio.
|
||||
|
||||
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
|
||||
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
|
||||
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
|
||||
path unchanged).
|
||||
"""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
tiles = batch.extra.pop(SR_TILES_KEY)
|
||||
plan = batch.extra.pop(SR_PLAN_KEY)
|
||||
video = stitch_tiles(tiles, plan)
|
||||
del tiles
|
||||
|
||||
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
|
||||
if requested_hw is not None:
|
||||
requested_h, requested_w = requested_hw
|
||||
video = crop_to_hw(
|
||||
video,
|
||||
(requested_h * plan.tiling_scale, requested_w * plan.tiling_scale),
|
||||
)
|
||||
|
||||
target_hw = resolve_target_hw(
|
||||
batch.sr_target_resolution,
|
||||
tuple(video.shape[-2:]),
|
||||
batch.sr_target_resize_mode,
|
||||
)
|
||||
if target_hw is not None:
|
||||
video = resize_to_target(video, target_hw)
|
||||
batch.height, batch.width = video.shape[-2:]
|
||||
return OutputBatch(
|
||||
output=to_output_video(video),
|
||||
audio=batch.audio,
|
||||
audio_sample_rate=batch.audio_sample_rate,
|
||||
fps=batch.fps,
|
||||
metrics=batch.metrics,
|
||||
usage=batch.usage,
|
||||
)
|
||||
+55
-135
@@ -16,7 +16,6 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
PiflowParams,
|
||||
SamplingSpec,
|
||||
module_dtype,
|
||||
)
|
||||
@@ -31,10 +30,7 @@ SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4)
|
||||
SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path)
|
||||
SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan
|
||||
SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb]
|
||||
# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out
|
||||
# of the single combined stage so the denoising stage can subclass the shared DenoisingStage
|
||||
# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain
|
||||
# data, like TilePlan above), and the chunk lists are CPU tensors throughout.
|
||||
# stages exchange immutable specs and CPU chunks, not resident model objects
|
||||
SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec
|
||||
SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec
|
||||
SR_CHUNKS_KEY = (
|
||||
@@ -43,11 +39,7 @@ SR_CHUNKS_KEY = (
|
||||
SR_DENOISED_KEY = (
|
||||
"kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise)
|
||||
)
|
||||
# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the
|
||||
# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale
|
||||
# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The
|
||||
# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to
|
||||
# (H * scale, W * scale) so the delivered video matches what was actually asked for.
|
||||
# original (H, W), used to crop VAE alignment padding from the stitched output
|
||||
SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw"
|
||||
|
||||
|
||||
@@ -71,7 +63,56 @@ def build_dit_spec(dit: Any) -> DitSpec:
|
||||
)
|
||||
|
||||
|
||||
def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]:
|
||||
def build_sampling_spec(
|
||||
*,
|
||||
arch: Kandinsky6SRArchConfig,
|
||||
tiling_scale: int,
|
||||
seed: int,
|
||||
num_steps: int,
|
||||
tiles_batch_size: int,
|
||||
tile_min_overlap: float,
|
||||
scheduler: FlowMatchEulerDiscreteScheduler,
|
||||
) -> SamplingSpec:
|
||||
"""Validate the loaded scheduler and resolve DiT calls before encoding tiles."""
|
||||
if num_steps < 1:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_steps}"
|
||||
)
|
||||
if not isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
|
||||
raise ValueError(
|
||||
"Kandinsky6 SR requires a FlowMatchEulerDiscreteScheduler or "
|
||||
"PiflowScheduler from the checkpoint"
|
||||
)
|
||||
is_piflow = isinstance(scheduler, PiflowScheduler)
|
||||
n_grid = scheduler.config.n_grid if is_piflow else 1
|
||||
expected_width = arch.in_visual_dim * n_grid
|
||||
if arch.out_visual_dim != expected_width:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR: {type(scheduler).__name__} requires a {expected_width}-channel "
|
||||
f"head, but out_visual_dim={arch.out_visual_dim}; use the scheduler "
|
||||
"from the same checkpoint as the transformer"
|
||||
)
|
||||
if is_piflow:
|
||||
nfe = scheduler.config.nfe
|
||||
if nfe is None or nfe < 1:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR: PiflowScheduler requires nfe >= 1, got {nfe}"
|
||||
)
|
||||
if arch.sr_cap_noise_timestep and arch.effective_override("instruct_type") in (
|
||||
"noise",
|
||||
"hybrid",
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"piflow sampler does not support cap_noise_timestep for noise/hybrid instruct"
|
||||
)
|
||||
if num_steps != nfe:
|
||||
logger.warning(
|
||||
"Kandinsky6 SR: ignoring num_inference_steps=%d; the pi-Flow checkpoint "
|
||||
"requires %d DiT calls per tile",
|
||||
num_steps,
|
||||
nfe,
|
||||
)
|
||||
num_steps = nfe
|
||||
table = arch.sr_scale_factor
|
||||
visual_size = arch.sr_visual_size[0]
|
||||
if table is None:
|
||||
@@ -84,138 +125,17 @@ def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, floa
|
||||
raise ValueError(
|
||||
f"sr_scale_factor has no entry for visual size {visual_size}: {table}"
|
||||
)
|
||||
return tuple(float(value) for value in values)
|
||||
|
||||
|
||||
def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None:
|
||||
"""pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler)."""
|
||||
if not isinstance(scheduler, PiflowScheduler):
|
||||
return None
|
||||
config = scheduler.config
|
||||
if config["nfe"] is None:
|
||||
return None
|
||||
return PiflowParams(
|
||||
nfe=int(config["nfe"]),
|
||||
num_policy_substeps=int(config["num_policy_substeps"]),
|
||||
final_step_size_scale=float(config["final_step_size_scale"]),
|
||||
shift=float(config["shift"]),
|
||||
n_grid=int(config["n_grid"]),
|
||||
eps=float(config["eps"]),
|
||||
)
|
||||
|
||||
|
||||
def _describe_scheduler(scheduler: Any) -> str:
|
||||
if scheduler is None:
|
||||
return "missing"
|
||||
if isinstance(scheduler, PiflowScheduler):
|
||||
return "a PiflowScheduler without nfe"
|
||||
return f"a {type(scheduler).__name__}"
|
||||
|
||||
|
||||
def _check_sampler_fits_head(
|
||||
*, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any
|
||||
) -> None:
|
||||
"""Reject incompatible scheduler and DiT head widths before encoding tiles."""
|
||||
head, grid = arch.head_width, arch.base_out_visual_dim
|
||||
if piflow is None:
|
||||
if head != grid:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / "
|
||||
f"pi-Flow head: {grid} channels per latent), but the sampler resolved "
|
||||
"to flow-Euler: the scheduler component is "
|
||||
f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with "
|
||||
"nfe selects pi-Flow. Use the scheduler of the same repo as the "
|
||||
"transformer."
|
||||
)
|
||||
return
|
||||
if piflow.nfe < 1:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, "
|
||||
f"got nfe={piflow.nfe}."
|
||||
)
|
||||
if head != piflow.n_grid * grid:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid "
|
||||
f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = "
|
||||
f"{piflow.n_grid * grid} channel DX head, but the transformer head is "
|
||||
f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The "
|
||||
"scheduler and the transformer come from different repos."
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_spec(
|
||||
*,
|
||||
arch: Kandinsky6SRArchConfig,
|
||||
tiling_scale: int,
|
||||
seed: int,
|
||||
num_steps: int,
|
||||
tiles_batch_size: int,
|
||||
tile_min_overlap: float,
|
||||
scheduler: Any = None,
|
||||
) -> SamplingSpec:
|
||||
"""Resolve sampling from the bundle scheduler, falling back to legacy DiT fields.
|
||||
|
||||
num_steps counts DiT calls, not Diffusers timestep grid points."""
|
||||
piflow = piflow_params_from_scheduler(scheduler)
|
||||
if piflow is None and arch.is_piflow:
|
||||
piflow = PiflowParams(
|
||||
nfe=arch.piflow_nfe,
|
||||
num_policy_substeps=arch.piflow_num_policy_substeps,
|
||||
final_step_size_scale=arch.piflow_final_step_size_scale,
|
||||
shift=arch.piflow_shift,
|
||||
n_grid=arch.n_grid,
|
||||
eps=arch.piflow_eps,
|
||||
)
|
||||
_check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler)
|
||||
return SamplingSpec(
|
||||
tiling_scale=tiling_scale,
|
||||
tiles_batch_size=tiles_batch_size,
|
||||
seed=seed,
|
||||
num_steps=num_steps,
|
||||
is_piflow=is_piflow,
|
||||
tile_min_overlap=tile_min_overlap,
|
||||
visual_size=arch.sr_visual_size[0],
|
||||
scale_factor=_rope_scale_factor(arch),
|
||||
scheduler_scale=arch.sr_scheduler_scale,
|
||||
visual_size=visual_size,
|
||||
scale_factor=tuple(float(value) for value in values),
|
||||
lq_noise_scale=arch.sr_lq_noise_scale,
|
||||
lq_noise_type=arch.sr_lq_noise_type,
|
||||
lq_channel_noise_scale=arch.sr_lq_channel_noise_scale,
|
||||
cap_noise_timestep=arch.sr_cap_noise_timestep,
|
||||
piflow=piflow,
|
||||
)
|
||||
|
||||
|
||||
def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any:
|
||||
"""Use the loaded scheduler when compatible; synthesize one for legacy configs."""
|
||||
if spec.piflow is not None:
|
||||
if isinstance(scheduler, PiflowScheduler):
|
||||
return scheduler
|
||||
return PiflowScheduler(
|
||||
nfe=spec.piflow.nfe,
|
||||
n_grid=spec.piflow.n_grid,
|
||||
shift=spec.piflow.shift,
|
||||
eps=spec.piflow.eps,
|
||||
final_step_size_scale=spec.piflow.final_step_size_scale,
|
||||
num_policy_substeps=spec.piflow.num_policy_substeps,
|
||||
)
|
||||
if isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
|
||||
return scheduler
|
||||
return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale)
|
||||
|
||||
|
||||
def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None:
|
||||
"""Validate the requested DiT-call count before denoising.
|
||||
|
||||
Diffusers counts timestep grid points instead: its default 5 is 4 calls here."""
|
||||
if num_inference_steps < 1:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}"
|
||||
)
|
||||
if spec.piflow is not None and num_inference_steps != spec.piflow.nfe:
|
||||
logger.warning(
|
||||
"Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow "
|
||||
"bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow "
|
||||
"bundles and every tile will run exactly nfe=%d DiT calls instead.",
|
||||
num_inference_steps,
|
||||
spec.piflow.nfe,
|
||||
spec.piflow.nfe,
|
||||
)
|
||||
|
||||
+79
-137
@@ -1,191 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Scheduler-driven denoising for Kandinsky 6 SR tile chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output
|
||||
DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor]
|
||||
StepCallback = Callable[[], None]
|
||||
StepContext = Callable[[int], AbstractContextManager]
|
||||
|
||||
|
||||
class PiflowParams(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""pi-Flow parameters from the scheduler or legacy flat DiT config."""
|
||||
|
||||
nfe: int
|
||||
num_policy_substeps: int
|
||||
final_step_size_scale: float
|
||||
shift: float
|
||||
n_grid: int
|
||||
eps: float
|
||||
|
||||
|
||||
class DitSpec(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Post-override behaviour of the DiT that the sampling code needs."""
|
||||
"""Post-override input layout and loaded parameter dtype."""
|
||||
|
||||
instruct_type: str | None
|
||||
visual_cond: bool
|
||||
in_visual_dim: int
|
||||
use_motion_score: bool
|
||||
patch_size: tuple[int, int, int]
|
||||
# The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the
|
||||
# LQ latent to this dtype before building the initial latent, so a bf16-loaded
|
||||
# checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT,
|
||||
# e.g. a mocked one in a test) keeps the LQ latent's own dtype.
|
||||
dtype: torch.dtype | None = None
|
||||
|
||||
|
||||
class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Everything besides the components that defines one tiled SR run."""
|
||||
"""Serializable settings shared by tile preparation and denoising."""
|
||||
|
||||
tiling_scale: int
|
||||
tiles_batch_size: int
|
||||
seed: int
|
||||
num_steps: int
|
||||
is_piflow: bool
|
||||
tile_min_overlap: float
|
||||
visual_size: int
|
||||
scale_factor: tuple[float, float, float]
|
||||
scheduler_scale: float
|
||||
lq_noise_scale: float
|
||||
lq_noise_type: str
|
||||
lq_channel_noise_scale: float
|
||||
cap_noise_timestep: bool
|
||||
piflow: PiflowParams | None
|
||||
|
||||
@property
|
||||
def steps_per_chunk(self) -> int:
|
||||
"""DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``."""
|
||||
return self.piflow.nfe if self.piflow is not None else self.num_steps
|
||||
|
||||
|
||||
def bf16_autocast(device: torch.device | str) -> AbstractContextManager:
|
||||
"""CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models."""
|
||||
if torch.device(device).type != "cuda":
|
||||
return nullcontext()
|
||||
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
|
||||
|
||||
|
||||
def module_dtype(module: torch.nn.Module) -> torch.dtype | None:
|
||||
"""Dtype of the first parameter (``None`` for a parameter-less module)."""
|
||||
return next((param.dtype for param in module.parameters()), None)
|
||||
|
||||
|
||||
def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor:
|
||||
"""Cast a floating tensor to the module's parameter dtype (reference helper)."""
|
||||
dtype = module_dtype(module)
|
||||
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
|
||||
return value.to(dtype=dtype)
|
||||
return value
|
||||
|
||||
|
||||
def euler_start_timestep(
|
||||
*, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None
|
||||
) -> float:
|
||||
"""Euler start time: the LQ noise level when capped for noise-type conditioning."""
|
||||
capped = cap_noise_timestep and instruct_type in (
|
||||
"noise",
|
||||
"hybrid",
|
||||
"hybrid_anchor",
|
||||
return (
|
||||
value.to(dtype=dtype)
|
||||
if dtype is not None and value.is_floating_point()
|
||||
else value
|
||||
)
|
||||
return lq_noise_scale if capped else 1.0
|
||||
|
||||
|
||||
def check_sampler_options(
|
||||
*, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None
|
||||
) -> None:
|
||||
"""Reject option combinations the reference sampler rejects."""
|
||||
if (
|
||||
piflow is not None
|
||||
and cap_noise_timestep
|
||||
and instruct_type in ("noise", "hybrid")
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"piflow sampler does not support cap_noise_timestep for noise/hybrid "
|
||||
"instruct"
|
||||
)
|
||||
|
||||
|
||||
def make_dit_fn(
|
||||
dit: Callable[..., torch.Tensor],
|
||||
*,
|
||||
latent_frames_hw: tuple[int, int, int],
|
||||
patch_size: tuple[int, int, int],
|
||||
scale_factor: tuple[float, float, float],
|
||||
use_motion_score: bool,
|
||||
step_context: StepContext | None = None,
|
||||
) -> DitFn:
|
||||
"""Bind per-chunk RoPE/motion inputs and an optional per-step context."""
|
||||
frames, height, width = latent_frames_hw
|
||||
context = step_context or (lambda step: nullcontext())
|
||||
|
||||
def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor:
|
||||
rope_pos = [
|
||||
torch.arange(frames, device=x.device),
|
||||
torch.arange(height // patch_size[1], device=x.device),
|
||||
torch.arange(width // patch_size[2], device=x.device),
|
||||
]
|
||||
motion = (
|
||||
torch.full((1,), 900.0, device=x.device, dtype=x.dtype)
|
||||
if use_motion_score
|
||||
else None
|
||||
)
|
||||
with context(step):
|
||||
return dit(
|
||||
x,
|
||||
model_time,
|
||||
rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion,
|
||||
)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def denoise_with_scheduler(
|
||||
x: torch.Tensor,
|
||||
dit_fn: DitFn,
|
||||
scheduler: Any,
|
||||
*,
|
||||
channels: int,
|
||||
is_piflow: bool,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Denoise one chunk after set_timesteps; keep conditioning channels fixed.
|
||||
|
||||
Promote predictions to fp32 so flow-Euler does not downcast its running
|
||||
state to the bf16 prediction dtype. pi-Flow already preserves the state dtype."""
|
||||
state = x[..., :channels].float()
|
||||
cond = x[..., channels:]
|
||||
for step, t in enumerate(scheduler.timesteps):
|
||||
model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0])
|
||||
hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
|
||||
prediction = dit_fn(hidden_states, model_time, step)
|
||||
if not is_piflow:
|
||||
prediction = prediction.float()
|
||||
state = scheduler.step(prediction, t, state, return_dict=False)[0]
|
||||
if on_step is not None:
|
||||
on_step()
|
||||
def _denoise_chunk(x, dit, scheduler, dit_spec, spec, step_context, on_step):
|
||||
# each chunk starts at step zero and releases its GPU temporaries before the next
|
||||
scheduler._step_index = None
|
||||
scheduler.set_begin_index(0)
|
||||
state = x[..., : dit_spec.in_visual_dim].float()
|
||||
cond = x[..., dit_spec.in_visual_dim :]
|
||||
frames, height, width = x.shape[1:4]
|
||||
rope_pos = [
|
||||
torch.arange(frames, device=x.device),
|
||||
torch.arange(height // dit_spec.patch_size[1], device=x.device),
|
||||
torch.arange(width // dit_spec.patch_size[2], device=x.device),
|
||||
]
|
||||
motion = (
|
||||
torch.full((1,), 900.0, device=x.device, dtype=state.dtype)
|
||||
if dit_spec.use_motion_score and not spec.is_piflow
|
||||
else None
|
||||
)
|
||||
with bf16_autocast(x.device):
|
||||
for step, t in enumerate(scheduler.timesteps):
|
||||
model_time = t.to(device=x.device, dtype=torch.float32).expand(
|
||||
state.shape[0]
|
||||
)
|
||||
hidden_states = (
|
||||
torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
|
||||
)
|
||||
with step_context(step) if step_context is not None else nullcontext():
|
||||
prediction = dit(
|
||||
hidden_states,
|
||||
model_time,
|
||||
rope_pos,
|
||||
scale_factor=spec.scale_factor,
|
||||
motion_score=motion,
|
||||
)
|
||||
# Euler otherwise downcasts the running state to the prediction dtype
|
||||
if not spec.is_piflow:
|
||||
prediction = prediction.float()
|
||||
state = scheduler.step(prediction, t, state, return_dict=False)[0]
|
||||
if on_step is not None:
|
||||
on_step()
|
||||
return state.float()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DitFn",
|
||||
"DitSpec",
|
||||
"PiflowParams",
|
||||
"SamplingSpec",
|
||||
"StepCallback",
|
||||
"StepContext",
|
||||
"bf16_autocast",
|
||||
"cast_to_module_dtype",
|
||||
"check_sampler_options",
|
||||
"denoise_with_scheduler",
|
||||
"euler_start_timestep",
|
||||
"make_dit_fn",
|
||||
"module_dtype",
|
||||
]
|
||||
def denoise_chunks(
|
||||
chunks: Sequence[torch.Tensor],
|
||||
dit: Callable[..., torch.Tensor],
|
||||
scheduler: Any,
|
||||
*,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
step_context: Callable[[int], AbstractContextManager] | None = None,
|
||||
on_step: Callable[[], None] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Denoise CPU chunks sequentially, preserving conditioning and fp32 state."""
|
||||
if spec.is_piflow:
|
||||
scheduler.set_timesteps(spec.num_steps, device=device)
|
||||
else:
|
||||
capped = spec.cap_noise_timestep and dit_spec.instruct_type in (
|
||||
"noise",
|
||||
"hybrid",
|
||||
"hybrid_anchor",
|
||||
)
|
||||
start = spec.lq_noise_scale if capped else 1.0
|
||||
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
|
||||
results = []
|
||||
for chunk in chunks:
|
||||
if not spec.is_piflow:
|
||||
scheduler.set_timesteps(sigmas=sigmas, device=device)
|
||||
results.append(
|
||||
_denoise_chunk(
|
||||
chunk.to(device), dit, scheduler, dit_spec, spec, step_context, on_step
|
||||
).cpu()
|
||||
)
|
||||
return results
|
||||
|
||||
+463
@@ -0,0 +1,463 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR pipeline stages, grouped by component while retaining separate offload lifetimes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
resolve_target_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_local_torch_device,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
has_decode_parallel_world,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_CHUNKS_KEY,
|
||||
SR_DENOISED_KEY,
|
||||
SR_DIT_SPEC_KEY,
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_PLAN_KEY,
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_SAMPLING_SPEC_KEY,
|
||||
SR_TILES_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
uses_latent_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
denoise_chunks,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
decode_chunks,
|
||||
encode_pixel_tile,
|
||||
encode_video_to_lr_latent,
|
||||
plan_tiles,
|
||||
prepare_tile_latents,
|
||||
stitch_tiles,
|
||||
upscale_lr_latent_tile,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
VAE_SPATIAL_FACTOR,
|
||||
pad_to_spatial_factor,
|
||||
pre_upscale_video,
|
||||
resolve_scale_request,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
SOURCE_AUDIO_SAMPLE_RATE,
|
||||
DecodedClip,
|
||||
decode_clip,
|
||||
extract_source_audio,
|
||||
resize_to_target,
|
||||
synthetic_clip,
|
||||
to_output_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
precision_to_dtype,
|
||||
resolve_precision,
|
||||
)
|
||||
|
||||
WARMUP_CLIP_FRAMES = 9
|
||||
WARMUP_CLIP_HW = (256, 384)
|
||||
|
||||
|
||||
class Kandinsky6SRInputStage(PipelineStage):
|
||||
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
|
||||
|
||||
Record requested dimensions before VAE padding so output can remove it."""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
|
||||
if batch.is_warmup:
|
||||
height, width = WARMUP_CLIP_HW
|
||||
clip = DecodedClip(
|
||||
frames=synthetic_clip(
|
||||
height=height, width=width, frames=WARMUP_CLIP_FRAMES
|
||||
),
|
||||
fps=24,
|
||||
source_fps=24.0,
|
||||
)
|
||||
else:
|
||||
path = batch.sampling_params.source_video_path()
|
||||
if path is None or not os.path.isfile(path):
|
||||
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
|
||||
self.log_info("Decoding %s", path)
|
||||
clip = decode_clip(path)
|
||||
video = clip.frames
|
||||
if pre_upscale != 1.0:
|
||||
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
|
||||
frames = video.shape[0]
|
||||
# pad every scale for the VAE; output crops padding without resizing source pixels
|
||||
video, (requested_h, requested_w) = pad_to_spatial_factor(
|
||||
video, VAE_SPATIAL_FACTOR
|
||||
)
|
||||
|
||||
batch.extra[SR_VIDEO_KEY] = video
|
||||
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
|
||||
batch.fps = clip.fps
|
||||
batch.num_frames = frames
|
||||
batch.height = requested_h * tiling_scale
|
||||
batch.width = requested_w * tiling_scale
|
||||
if not batch.is_warmup:
|
||||
audio = extract_source_audio(
|
||||
path,
|
||||
sample_rate=SOURCE_AUDIO_SAMPLE_RATE,
|
||||
max_seconds=frames / clip.fps,
|
||||
)
|
||||
batch.audio = torch.from_numpy(audio) if audio is not None else None
|
||||
batch.audio_sample_rate = (
|
||||
SOURCE_AUDIO_SAMPLE_RATE if audio is not None else None
|
||||
)
|
||||
return batch
|
||||
|
||||
|
||||
class Kandinsky6SREncodeStage(PipelineStage):
|
||||
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
|
||||
|
||||
def __init__(self, vae, latent_upscaler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
# Only consulted to pick the path (does the bank have this scale?).
|
||||
self.latent_upscaler = latent_upscaler
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="encode_video",
|
||||
target_dtype=vae_dtype,
|
||||
start_at_stage_entry=False,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
if not uses_latent_path(self.latent_upscaler, tiling_scale):
|
||||
return batch
|
||||
video = batch.extra.pop(SR_VIDEO_KEY)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="encode_video"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
lr_latent = encode_video_to_lr_latent(
|
||||
video, vae, device=get_local_torch_device()
|
||||
)
|
||||
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
|
||||
return batch
|
||||
|
||||
|
||||
class Kandinsky6SRLatentPrepStage(PipelineStage):
|
||||
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
|
||||
|
||||
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
self.transformer = transformer # arch-config / dtype only: never called here
|
||||
self.latent_upscaler = latent_upscaler
|
||||
self.scheduler = (
|
||||
scheduler # resolved sampler validation only: never stepped here
|
||||
)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
uses: list[ComponentUse] = []
|
||||
if self.latent_upscaler is not None:
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name,
|
||||
"latent_upscaler",
|
||||
phase="upscale_tiles",
|
||||
target_dtype=torch.bfloat16,
|
||||
)
|
||||
)
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
|
||||
)
|
||||
)
|
||||
return uses
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
arch = server_args.pipeline_config.dit_config.arch_config
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=tiling_scale,
|
||||
seed=seed,
|
||||
num_steps=batch.num_inference_steps,
|
||||
tiles_batch_size=batch.sr_tiles_batch_size,
|
||||
tile_min_overlap=batch.sr_tile_min_overlap,
|
||||
scheduler=self.scheduler,
|
||||
)
|
||||
dit_spec = build_dit_spec(self.transformer)
|
||||
device = get_local_torch_device()
|
||||
|
||||
latent_path = uses_latent_path(self.latent_upscaler, tiling_scale)
|
||||
source = batch.extra.pop(SR_LR_LATENT_KEY if latent_path else SR_VIDEO_KEY)
|
||||
factor = VAE_SPATIAL_FACTOR if latent_path else 1
|
||||
plan = plan_tiles(
|
||||
frame_hw=tuple(size * factor for size in source.shape[-2:]),
|
||||
visual_size=spec.visual_size,
|
||||
tiling_scale=tiling_scale,
|
||||
tile_min_overlap=spec.tile_min_overlap,
|
||||
)
|
||||
with self.use_declared_component(
|
||||
component_name="latent_upscaler" if latent_path else "vae",
|
||||
module=self.latent_upscaler if latent_path else self.vae,
|
||||
phase="upscale_tiles" if latent_path else "encode_tiles",
|
||||
) as component:
|
||||
assert component is not None
|
||||
if latent_path:
|
||||
self.latent_upscaler = component
|
||||
tile_encoder = partial(
|
||||
upscale_lr_latent_tile,
|
||||
upscale_fn=partial(component.upscale, scale=tiling_scale),
|
||||
lu_dtype=module_dtype(component),
|
||||
scaling_factor=self.vae.scaling_factor,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.vae = component
|
||||
tile_encoder = partial(
|
||||
encode_pixel_tile,
|
||||
vae=component,
|
||||
scaling_factor=component.scaling_factor,
|
||||
device=device,
|
||||
)
|
||||
chunks = prepare_tile_latents(
|
||||
source,
|
||||
plan,
|
||||
tile_encoder=tile_encoder,
|
||||
latent_path=latent_path,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
batch.extra[SR_PLAN_KEY] = plan
|
||||
batch.extra[SR_CHUNKS_KEY] = chunks
|
||||
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
|
||||
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
|
||||
return batch
|
||||
|
||||
|
||||
class Kandinsky6SRDenoisingStage(DenoisingStage):
|
||||
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
|
||||
|
||||
def _owns_compile_warmup_lifecycle(self) -> bool:
|
||||
# forward wraps the tiled loop in the shared compile warmup lifecycle
|
||||
return True
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_SAMPLING_SPEC_KEY]",
|
||||
batch.extra.get(SR_SAMPLING_SPEC_KEY),
|
||||
V.not_none,
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
dit_dtype = precision_to_dtype(
|
||||
server_args.pipeline_config.dit_precision, "dit_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"transformer",
|
||||
phase="denoise_tiles",
|
||||
target_dtype=dit_dtype,
|
||||
preferred_ready_after_request=True,
|
||||
memory_intensive=True,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
with self._offload_for_torch_compile_warmup(batch):
|
||||
return self._denoise_tiles(batch, server_args)
|
||||
|
||||
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
|
||||
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
|
||||
chunks = batch.extra.pop(SR_CHUNKS_KEY)
|
||||
device = get_local_torch_device()
|
||||
|
||||
self._maybe_enable_cache_dit_and_torch_compile(spec.num_steps, batch)
|
||||
|
||||
with self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer, phase="denoise_tiles"
|
||||
) as transformer:
|
||||
assert transformer is not None
|
||||
self.transformer = transformer
|
||||
total = spec.num_steps * len(chunks)
|
||||
with self.progress_bar(
|
||||
total=total, batch=batch, desc="Kandinsky6 SR denoising"
|
||||
) as progress:
|
||||
denoised = denoise_chunks(
|
||||
chunks,
|
||||
transformer,
|
||||
self.scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
step_context=lambda step: set_forward_context(
|
||||
current_timestep=step, attn_metadata=None, forward_batch=batch
|
||||
),
|
||||
on_step=progress.update,
|
||||
)
|
||||
self._finish_active_component_use()
|
||||
batch.extra[SR_DENOISED_KEY] = denoised
|
||||
return batch
|
||||
|
||||
|
||||
class Kandinsky6SRDecodeStage(PipelineStage):
|
||||
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
|
||||
|
||||
def __init__(self, vae) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="decode_tiles",
|
||||
target_dtype=vae_dtype,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
denoised = batch.extra.pop(SR_DENOISED_KEY)
|
||||
device = get_local_torch_device()
|
||||
group = None
|
||||
if (
|
||||
server_args.pipeline_config.vae_config.use_parallel_tiling
|
||||
and has_decode_parallel_world()
|
||||
and not can_install_spatial_shard_parallel_decode(
|
||||
server_args.pipeline_config.vae_config
|
||||
)
|
||||
):
|
||||
group = get_decode_parallel_group_coordinator()
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="decode_tiles"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
tiles = decode_chunks(
|
||||
denoised,
|
||||
vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
device=device,
|
||||
group=group,
|
||||
)
|
||||
batch.extra[SR_TILES_KEY] = tiles
|
||||
return batch
|
||||
|
||||
|
||||
class Kandinsky6SROutputStage(PipelineStage):
|
||||
"""Hann-blends the decoded tiles, restores the requested output size, applies the
|
||||
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
|
||||
plus the mono source audio.
|
||||
|
||||
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
|
||||
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
|
||||
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
|
||||
path unchanged).
|
||||
"""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
tiles = batch.extra.pop(SR_TILES_KEY)
|
||||
plan = batch.extra.pop(SR_PLAN_KEY)
|
||||
video = stitch_tiles(tiles, plan)
|
||||
del tiles
|
||||
|
||||
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
|
||||
if requested_hw is not None:
|
||||
requested_h, requested_w = requested_hw
|
||||
video = video[
|
||||
...,
|
||||
: requested_h * plan.tiling_scale,
|
||||
: requested_w * plan.tiling_scale,
|
||||
]
|
||||
|
||||
target_hw = resolve_target_hw(
|
||||
batch.sr_target_resolution,
|
||||
tuple(video.shape[-2:]),
|
||||
batch.sr_target_resize_mode,
|
||||
)
|
||||
if target_hw is not None:
|
||||
video = resize_to_target(video, target_hw)
|
||||
batch.height, batch.width = video.shape[-2:]
|
||||
return OutputBatch(
|
||||
output=to_output_video(video),
|
||||
audio=batch.audio,
|
||||
audio_sample_rate=batch.audio_sample_rate,
|
||||
fps=batch.fps,
|
||||
metrics=batch.metrics,
|
||||
usage=batch.usage,
|
||||
)
|
||||
+41
-179
@@ -6,10 +6,10 @@ Each chunk uses seed + its first tile index, matching the reference.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
from torch.nn import functional as F
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import (
|
||||
@@ -18,14 +18,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
StepCallback,
|
||||
StepContext,
|
||||
bf16_autocast,
|
||||
cast_to_module_dtype,
|
||||
check_sampler_options,
|
||||
denoise_with_scheduler,
|
||||
euler_start_timestep,
|
||||
make_dit_fn,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
@@ -36,7 +30,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
extract_all_tiles,
|
||||
latent_tile_grid_from_pixel_grid,
|
||||
stitch_tiles_hanning,
|
||||
upsample_tiles_to_base,
|
||||
)
|
||||
|
||||
LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor]
|
||||
@@ -147,188 +140,57 @@ def encode_pixel_tile(
|
||||
return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor
|
||||
|
||||
|
||||
def build_chunk_latent(
|
||||
lq_tiles: Sequence[torch.Tensor],
|
||||
*,
|
||||
seed: int,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles."""
|
||||
lq = torch.cat(list(lq_tiles), dim=0)
|
||||
batch = len(lq_tiles)
|
||||
x = build_initial_latent(
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
visual_cond=dit_spec.visual_cond,
|
||||
in_visual_dim=dit_spec.in_visual_dim,
|
||||
lq_latent=lq,
|
||||
batch_size=batch,
|
||||
duration=lq.shape[0] // batch,
|
||||
height=lq.shape[1],
|
||||
width=lq.shape[2],
|
||||
device=device,
|
||||
seed=seed,
|
||||
lq_noise_scale=spec.lq_noise_scale,
|
||||
lq_noise_type=spec.lq_noise_type,
|
||||
lq_channel_noise_scale=spec.lq_channel_noise_scale,
|
||||
# The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before
|
||||
# building the initial latent), not a hard-coded fp32 regardless of precision.
|
||||
dtype=dit_spec.dtype,
|
||||
)
|
||||
return x.reshape(batch, -1, *x.shape[1:])
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_lu_tile_latents(
|
||||
lr_latent: torch.Tensor,
|
||||
def prepare_tile_latents(
|
||||
source: torch.Tensor,
|
||||
plan: TilePlan,
|
||||
*,
|
||||
upscale_fn: LatentUpscaleFn,
|
||||
lu_dtype: torch.dtype | None,
|
||||
scaling_factor: float,
|
||||
tile_encoder: Callable[[torch.Tensor], torch.Tensor],
|
||||
latent_path: bool,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
spatial_factor: int = VAE_SPATIAL_FACTOR,
|
||||
) -> list[torch.Tensor]:
|
||||
"""LU path: initial latents (CPU) of every chunk from the whole-video latent."""
|
||||
latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
|
||||
tiles = extract_all_tiles(lr_latent, latent_grid)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
|
||||
lq_tiles = [
|
||||
upscale_lr_latent_tile(
|
||||
tile,
|
||||
upscale_fn,
|
||||
lu_dtype=lu_dtype,
|
||||
scaling_factor=scaling_factor,
|
||||
device=device,
|
||||
)
|
||||
for tile in tiles[start:stop]
|
||||
]
|
||||
chunk = build_chunk_latent(
|
||||
lq_tiles,
|
||||
seed=spec.seed + start,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
chunks.append(chunk.cpu())
|
||||
return chunks
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_pixel_tile_latents(
|
||||
video: torch.Tensor,
|
||||
plan: TilePlan,
|
||||
*,
|
||||
vae: torch.nn.Module,
|
||||
scaling_factor: float,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk."""
|
||||
tiles = extract_all_tiles(video, plan.pixel_grid)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
|
||||
enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw)
|
||||
lq_tiles = [
|
||||
encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device)
|
||||
for tile in enlarged
|
||||
]
|
||||
chunk = build_chunk_latent(
|
||||
lq_tiles,
|
||||
seed=spec.seed + start,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
chunks.append(chunk.cpu())
|
||||
return chunks
|
||||
|
||||
|
||||
def reset_scheduler_for_chunk(scheduler: Any) -> None:
|
||||
"""Reset step and begin indices so each tile chunk starts at the same timestep."""
|
||||
scheduler._step_index = None
|
||||
scheduler.set_begin_index(0)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def denoise_chunk(
|
||||
x: torch.Tensor,
|
||||
dit: Callable[..., torch.Tensor],
|
||||
scheduler: Any,
|
||||
*,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
step_context: StepContext | None = None,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler."""
|
||||
is_piflow = spec.piflow is not None
|
||||
check_sampler_options(
|
||||
piflow=spec.piflow,
|
||||
cap_noise_timestep=spec.cap_noise_timestep,
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
"""Encode/upscale one tile batch at a time; keep prepared chunks on CPU."""
|
||||
grid = (
|
||||
latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
|
||||
if latent_path
|
||||
else plan.pixel_grid
|
||||
)
|
||||
if not is_piflow:
|
||||
# The Euler start timestep is only meaningful for the scheduler this chunk is about to
|
||||
# run: re-derive it instead of baking a stale one into ``set_timesteps`` up front.
|
||||
start = euler_start_timestep(
|
||||
cap_noise_timestep=spec.cap_noise_timestep,
|
||||
lq_noise_scale=spec.lq_noise_scale,
|
||||
tiles = extract_all_tiles(source, grid)
|
||||
chunks = []
|
||||
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
|
||||
inputs = tiles[start:stop]
|
||||
if not latent_path:
|
||||
inputs = [
|
||||
F.interpolate(
|
||||
tile.float(),
|
||||
size=plan.base_hw,
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
).permute(0, 2, 3, 1)
|
||||
for tile in inputs
|
||||
]
|
||||
lq_tiles = [tile_encoder(tile) for tile in inputs]
|
||||
lq = torch.cat(lq_tiles, dim=0)
|
||||
x = build_initial_latent(
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
visual_cond=dit_spec.visual_cond,
|
||||
in_visual_dim=dit_spec.in_visual_dim,
|
||||
lq_latent=lq,
|
||||
device=device,
|
||||
seed=spec.seed + start,
|
||||
lq_noise_scale=spec.lq_noise_scale,
|
||||
lq_noise_type=spec.lq_noise_type,
|
||||
lq_channel_noise_scale=spec.lq_channel_noise_scale,
|
||||
# conditioning noise must round in the loaded DiT dtype
|
||||
dtype=dit_spec.dtype,
|
||||
)
|
||||
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
|
||||
scheduler.set_timesteps(sigmas=sigmas, device=x.device)
|
||||
reset_scheduler_for_chunk(scheduler)
|
||||
dit_fn = make_dit_fn(
|
||||
dit,
|
||||
latent_frames_hw=tuple(x.shape[1:4]),
|
||||
patch_size=dit_spec.patch_size,
|
||||
scale_factor=spec.scale_factor,
|
||||
use_motion_score=dit_spec.use_motion_score and not is_piflow,
|
||||
step_context=step_context,
|
||||
)
|
||||
with bf16_autocast(x.device):
|
||||
return denoise_with_scheduler(
|
||||
x,
|
||||
dit_fn,
|
||||
scheduler,
|
||||
channels=dit_spec.in_visual_dim,
|
||||
is_piflow=is_piflow,
|
||||
on_step=on_step,
|
||||
)
|
||||
|
||||
|
||||
def denoise_chunks(
|
||||
chunks: Sequence[torch.Tensor],
|
||||
dit: Callable[..., torch.Tensor],
|
||||
scheduler: Any,
|
||||
*,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
step_context: StepContext | None = None,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Denoise chunks sequentially on device and return CPU latents."""
|
||||
if spec.piflow is not None:
|
||||
scheduler.set_timesteps(spec.piflow.nfe, device=device)
|
||||
return [
|
||||
denoise_chunk(
|
||||
chunk.to(device),
|
||||
dit,
|
||||
scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
step_context=step_context,
|
||||
on_step=on_step,
|
||||
).cpu()
|
||||
for chunk in chunks
|
||||
]
|
||||
chunk = x.reshape(len(lq_tiles), -1, *x.shape[1:])
|
||||
del lq, x
|
||||
chunks.append(chunk.cpu())
|
||||
return chunks
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
|
||||
+9
-38
@@ -58,23 +58,6 @@ def extract_all_tiles(
|
||||
return tiles
|
||||
|
||||
|
||||
def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize Hann-weighted tiles without flooring positive weights.
|
||||
|
||||
Corner weights are tiny but valid; clamping them darkens tile boundaries.
|
||||
Pixels outside all tiles have zero weight and remain zero."""
|
||||
covered = weight > 0
|
||||
safe_weight = torch.where(covered, weight, torch.ones_like(weight))
|
||||
return torch.where(covered, acc / safe_weight, torch.zeros_like(acc))
|
||||
|
||||
|
||||
def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor:
|
||||
"""Create a [h, w] Hann window with nonzero endpoints."""
|
||||
wy = torch.hann_window(h + 2, device=device)[1:-1]
|
||||
wx = torch.hann_window(w + 2, device=device)[1:-1]
|
||||
return wy[:, None] * wx[None, :]
|
||||
|
||||
|
||||
def stitch_tiles_hanning(
|
||||
tiles: list[torch.Tensor],
|
||||
grid: TileGrid,
|
||||
@@ -88,8 +71,11 @@ def stitch_tiles_hanning(
|
||||
hr_tile_h, hr_tile_w = first.shape[2], first.shape[3]
|
||||
device = first.device
|
||||
|
||||
window = hanning_window_2d(hr_tile_h, hr_tile_w, device)
|
||||
window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w]
|
||||
# nonzero endpoints keep the outermost tile pixels covered
|
||||
wy = torch.hann_window(hr_tile_h + 2, device=device)[1:-1]
|
||||
wx = torch.hann_window(hr_tile_w + 2, device=device)[1:-1]
|
||||
window = (wy[:, None] * wx[None, :])[None, None]
|
||||
del wy, wx
|
||||
|
||||
out_h = original_h * scale
|
||||
out_w = original_w * scale
|
||||
@@ -106,7 +92,10 @@ def stitch_tiles_hanning(
|
||||
weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window
|
||||
tile_idx += 1
|
||||
|
||||
return _normalize_blend(pred_acc, weight_acc)
|
||||
# preserve tiny positive corner weights; only uncovered pixels stay zero
|
||||
covered = weight_acc > 0
|
||||
safe_weight = torch.where(covered, weight_acc, torch.ones_like(weight_acc))
|
||||
return torch.where(covered, pred_acc / safe_weight, torch.zeros_like(pred_acc))
|
||||
|
||||
|
||||
def axis_positions_even(
|
||||
@@ -179,18 +168,6 @@ def closest_base_resolution(
|
||||
return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio))
|
||||
|
||||
|
||||
def upsample_tiles_to_base(
|
||||
raw_tiles: list[torch.Tensor], base_h: int, base_w: int
|
||||
) -> list[torch.Tensor]:
|
||||
"""Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``."""
|
||||
return [
|
||||
functional.interpolate(
|
||||
tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False
|
||||
).permute(0, 2, 3, 1)
|
||||
for tile in raw_tiles
|
||||
]
|
||||
|
||||
|
||||
def resolve_scale_request(scale: float) -> tuple[int, float]:
|
||||
"""Decompose total scale into (integer tiling scale, pixel pre-upscale)."""
|
||||
requested = float(scale)
|
||||
@@ -240,9 +217,3 @@ def pad_to_spatial_factor(
|
||||
return video, (height, width)
|
||||
padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate")
|
||||
return padded, (height, width)
|
||||
|
||||
|
||||
def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""Crop [..., H, W] to hw from the top-left, undoing bottom/right padding."""
|
||||
height, width = hw
|
||||
return video[..., :height, :width]
|
||||
|
||||
+62
-23
@@ -1,24 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Streaming video / audio input and output helpers of Kandinsky 6 video SR.
|
||||
# Frame selection and resize adapted from the Kandinsky 6 SR inference reference
|
||||
# (k6_video / sr_core, Apache-2.0).
|
||||
"""Bounded-memory SR video/audio I/O, preserving reference frame selection and resize."""
|
||||
|
||||
Decoding follows the reference CLI (``read_video_tchw_uint8`` ->
|
||||
``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams:
|
||||
only the frames that survive fps resampling and the 121-frame cap are converted to RGB
|
||||
and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily
|
||||
(it is part of the ``diffusion`` extra).
|
||||
"""
|
||||
import logging
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import functional as F
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
MAX_NUM_FRAMES,
|
||||
RESAMPLE_FPS_TOLERANCE,
|
||||
TARGET_FPS,
|
||||
clip_to_aligned_frames,
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
ASPECT_MISMATCH_TOLERANCE,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
# trained on 5 s at 24 fps; select at most 121 frames, aligned to 1+8k
|
||||
TARGET_FPS = 24
|
||||
MAX_NUM_FRAMES = 121
|
||||
RESAMPLE_FPS_TOLERANCE = 1.5
|
||||
SOURCE_AUDIO_SAMPLE_RATE = 44100
|
||||
OUTPUT_FRAME_CHUNK = 8
|
||||
|
||||
@@ -32,16 +32,10 @@ class DecodedClip(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class FrameSelector:
|
||||
"""Streaming form of ``resample_to_target_fps`` + the first-121-frames cap.
|
||||
"""Stream reference indices round(i*step), capped at int(total/step) and max_frames.
|
||||
|
||||
Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the
|
||||
selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further
|
||||
frame can change the result, and :meth:`limit` trims the selection once the source
|
||||
length is known. For a source faster than the target the selected indices are
|
||||
``round(i * step)``, ``step = source_fps / target_fps``, of which there are
|
||||
``int(total / step)`` (those below ``total``), exactly as in the reference;
|
||||
otherwise every frame is selected.
|
||||
"""
|
||||
In-tolerance and slower sources keep every frame. Stop only once the source
|
||||
length guarantees that no retained frame will be trimmed by limit()."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -116,9 +110,9 @@ def decode_clip(
|
||||
kept = selector.limit(seen, len(frames))
|
||||
if kept == 0:
|
||||
raise ValueError(f"Video {path} has no readable frames.")
|
||||
video = torch.stack(frames[:kept]).contiguous()
|
||||
kept = 1 + 8 * ((kept - 1) // 8)
|
||||
return DecodedClip(
|
||||
frames=clip_to_aligned_frames(video, max_frames),
|
||||
frames=torch.stack(frames[:kept]).contiguous(),
|
||||
fps=selector.effective_fps,
|
||||
source_fps=source_fps,
|
||||
)
|
||||
@@ -188,3 +182,48 @@ def to_output_video(video: torch.Tensor) -> torch.Tensor:
|
||||
(chunk.float() + 0.5) / 255.0
|
||||
).to(torch.float16)
|
||||
return out
|
||||
|
||||
|
||||
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
|
||||
channels, frames, height, width = video.shape
|
||||
target_h, target_w = target_hw
|
||||
if (height, width) == (target_h, target_w):
|
||||
return video
|
||||
if target_h > height or target_w > width:
|
||||
raise ValueError(
|
||||
f"target {target_w}x{target_h} exceeds the SR result {width}x{height}; "
|
||||
"pick a lower tier or a higher --resolution-scale for this source."
|
||||
)
|
||||
source_ratio, target_ratio = width / height, target_w / target_h
|
||||
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
|
||||
logger.warning(
|
||||
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f; resizing "
|
||||
"anisotropically (no crop), the picture will be squeezed by %.1f%%.",
|
||||
width,
|
||||
height,
|
||||
source_ratio,
|
||||
target_w,
|
||||
target_h,
|
||||
target_ratio,
|
||||
abs(source_ratio / target_ratio - 1) * 100,
|
||||
)
|
||||
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
|
||||
for start in range(0, frames, OUTPUT_FRAME_CHUNK):
|
||||
chunk = video[:, start : start + OUTPUT_FRAME_CHUNK].permute(1, 0, 2, 3).float()
|
||||
resized = F.interpolate(
|
||||
chunk, size=target_hw, mode="bilinear", antialias=True, align_corners=False
|
||||
)
|
||||
out[:, start : start + OUTPUT_FRAME_CHUNK] = (
|
||||
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
|
||||
)
|
||||
logger.info(
|
||||
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
|
||||
width,
|
||||
height,
|
||||
target_w,
|
||||
target_h,
|
||||
width / target_w,
|
||||
height / target_h,
|
||||
)
|
||||
return out
|
||||
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
|
||||
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
|
||||
"""Frame selection and bounded-memory output resize for Kandinsky 6 SR."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
from torch.nn import functional as f # noqa: N812
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
ASPECT_MISMATCH_TOLERANCE,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) ---
|
||||
# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames
|
||||
# (= 1 + 8*15); feeding more frames or another rate is out of contract.
|
||||
TARGET_FPS: int = 24
|
||||
MAX_NUM_FRAMES: int = 121
|
||||
# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no
|
||||
# resample): covers 23.98 (NTSC) and 25.00 sources.
|
||||
RESAMPLE_FPS_TOLERANCE: float = 1.5
|
||||
# Frames per interpolate call when downscaling the output (a 2K clip is
|
||||
# ~2.4 GB as uint8, so the resize walks the clip in chunks).
|
||||
RESIZE_FRAME_CHUNK: int = 8
|
||||
|
||||
|
||||
def select_frame_indices(
|
||||
total_frames: int, src_fps: float, target_fps: float
|
||||
) -> list[int]:
|
||||
"""Select rounded frame indices to downsample src_fps to target_fps."""
|
||||
step = src_fps / target_fps
|
||||
indices = [round(i * step) for i in range(int(total_frames / step))]
|
||||
return [i for i in indices if i < total_frames]
|
||||
|
||||
|
||||
def resample_to_target_fps(
|
||||
video: torch.Tensor,
|
||||
src_fps: float,
|
||||
target_fps: int = TARGET_FPS,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Downsample high-fps inputs; retain near-target and lower-fps inputs.
|
||||
|
||||
Return [T, C, H, W] video and its effective playback rate."""
|
||||
if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE:
|
||||
return video, round(src_fps)
|
||||
if src_fps > target_fps:
|
||||
indices = select_frame_indices(video.shape[0], src_fps, target_fps)
|
||||
logger.warning(
|
||||
"Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); "
|
||||
"source temporal detail beyond %sfps is discarded.",
|
||||
src_fps,
|
||||
target_fps,
|
||||
video.shape[0],
|
||||
len(indices),
|
||||
target_fps,
|
||||
)
|
||||
return video[indices], target_fps
|
||||
logger.warning(
|
||||
"Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); "
|
||||
"output is mildly out of distribution.",
|
||||
src_fps,
|
||||
target_fps,
|
||||
)
|
||||
return video, round(src_fps)
|
||||
|
||||
|
||||
def clip_to_aligned_frames(
|
||||
video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES
|
||||
) -> torch.Tensor:
|
||||
"""Cap the frame count, then floor-align to 1+8k; reject empty video."""
|
||||
if max_num_frames > 0:
|
||||
video = video[:max_num_frames]
|
||||
aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0
|
||||
if aligned == 0:
|
||||
msg = "Video has no readable frames."
|
||||
raise ValueError(msg)
|
||||
return video[:aligned]
|
||||
|
||||
|
||||
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
|
||||
_c, _t, height, width = video.shape
|
||||
target_h, target_w = target_hw
|
||||
if (height, width) == (target_h, target_w):
|
||||
return video
|
||||
if target_h > height or target_w > width:
|
||||
msg = (
|
||||
f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would "
|
||||
f"upscale. Pick a lower tier or a higher --resolution-scale for this source."
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
source_ratio, target_ratio = width / height, target_w / target_h
|
||||
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
|
||||
logger.warning(
|
||||
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically "
|
||||
"(no crop), the picture will be squeezed by %.1f%%.",
|
||||
width,
|
||||
height,
|
||||
source_ratio,
|
||||
target_w,
|
||||
target_h,
|
||||
target_ratio,
|
||||
abs(source_ratio / target_ratio - 1) * 100,
|
||||
)
|
||||
|
||||
channels, frames = video.shape[0], video.shape[1]
|
||||
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
|
||||
for start in range(0, frames, RESIZE_FRAME_CHUNK):
|
||||
chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float()
|
||||
resized = f.interpolate(
|
||||
chunk,
|
||||
size=(target_h, target_w),
|
||||
mode="bilinear",
|
||||
antialias=True,
|
||||
align_corners=False,
|
||||
)
|
||||
out[:, start : start + RESIZE_FRAME_CHUNK] = (
|
||||
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
|
||||
width,
|
||||
height,
|
||||
target_w,
|
||||
target_h,
|
||||
width / target_w,
|
||||
height / target_h,
|
||||
)
|
||||
return out
|
||||
@@ -5,7 +5,28 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime import server_args as server_args_module
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def single_process_model_parallel():
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(get_free_port_below_ephemeral()),
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
patch.setenv(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
yield
|
||||
|
||||
|
||||
def _make_unit_server_args():
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -21,6 +22,7 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
@@ -37,29 +39,30 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
denoise_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
LatentUpscaleFn,
|
||||
decode_chunks,
|
||||
denoise_chunks,
|
||||
encode_pixel_tile,
|
||||
encode_video_to_lr_latent,
|
||||
plan_tiles,
|
||||
prepare_lu_tile_latents,
|
||||
prepare_pixel_tile_latents,
|
||||
prepare_tile_latents,
|
||||
stitch_tiles,
|
||||
upscale_lr_latent_tile,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
VAE_SPATIAL_FACTOR,
|
||||
)
|
||||
|
||||
# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels.
|
||||
# official transformer layout: total output width, scheduler configured separately
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
out_visual_dim=12,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
@@ -69,15 +72,16 @@ TINY_DIT = dict(
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
use_text=False,
|
||||
attribute_overrides=None,
|
||||
sr_params=dict(visual_size=[512], scale_factor={"512": [1.0, 1.0, 1.0]}),
|
||||
)
|
||||
TINY_PIFLOW = dict(
|
||||
n_grid=3,
|
||||
piflow_nfe=2,
|
||||
piflow_num_policy_substeps=8,
|
||||
piflow_final_step_size_scale=0.5,
|
||||
piflow_shift=5.0,
|
||||
piflow_eps=1e-6,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
nfe=2,
|
||||
num_policy_substeps=8,
|
||||
final_step_size_scale=0.5,
|
||||
shift=5.0,
|
||||
eps=1e-6,
|
||||
)
|
||||
TINY_KVAE = dict(
|
||||
ch=8,
|
||||
@@ -211,9 +215,7 @@ def super_resolve(
|
||||
) -> torch.Tensor:
|
||||
"""Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``.
|
||||
|
||||
``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the
|
||||
already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded
|
||||
component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits.
|
||||
``upscale_fn`` selects the LU path; without it the pixel path runs.
|
||||
"""
|
||||
device = torch.device(device)
|
||||
common = dict(
|
||||
@@ -230,30 +232,43 @@ def super_resolve(
|
||||
lr_latent.shape[-1] * spatial_factor,
|
||||
)
|
||||
plan = plan_tiles(frame_hw=frame_hw, **common)
|
||||
chunks = prepare_lu_tile_latents(
|
||||
lr_latent,
|
||||
plan,
|
||||
source = lr_latent
|
||||
tile_encoder = partial(
|
||||
upscale_lr_latent_tile,
|
||||
upscale_fn=upscale_fn,
|
||||
lu_dtype=lu_dtype,
|
||||
scaling_factor=scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
spatial_factor=spatial_factor,
|
||||
)
|
||||
else:
|
||||
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
|
||||
chunks = prepare_pixel_tile_latents(
|
||||
video,
|
||||
plan,
|
||||
source = video
|
||||
tile_encoder = partial(
|
||||
encode_pixel_tile,
|
||||
vae=vae,
|
||||
scaling_factor=scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
chunks = prepare_tile_latents(
|
||||
source,
|
||||
plan,
|
||||
tile_encoder=tile_encoder,
|
||||
latent_path=upscale_fn is not None,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
spatial_factor=spatial_factor,
|
||||
)
|
||||
denoised = denoise_chunks(
|
||||
chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device
|
||||
chunks,
|
||||
dit,
|
||||
scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
step_context=lambda step: set_forward_context(
|
||||
current_timestep=step, attn_metadata=None
|
||||
),
|
||||
)
|
||||
tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device)
|
||||
return stitch_tiles(tiles, plan)
|
||||
|
||||
@@ -15,6 +15,7 @@ reference).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
@@ -78,12 +79,16 @@ def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load():
|
||||
vae = Kandinsky6AudioVAE(config)
|
||||
assert vae.vae._weights_normalized is True
|
||||
|
||||
# `load_state_dict(vae.state_dict())` mirrors VAELoader's real call
|
||||
# (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's
|
||||
# own current weights standing in for "a checkpoint's own weights" --
|
||||
# exactly what happens when a real checkpoint's stored tensors are
|
||||
# loaded: no code path scales or otherwise disturbs them afterward.
|
||||
vae.load_state_dict(vae.state_dict())
|
||||
checkpoint = dict(vae.state_dict())
|
||||
checkpoint["vae.data_mean"] = torch.full((1, 80, 1), -2.5)
|
||||
checkpoint["vae.data_std"] = torch.full((1, 80, 1), 1.25)
|
||||
vae.load_state_dict(checkpoint, strict=True)
|
||||
torch.testing.assert_close(vae.vae.data_mean, checkpoint["vae.data_mean"])
|
||||
torch.testing.assert_close(vae.vae.data_std, checkpoint["vae.data_std"])
|
||||
for key in ("vae.data_mean", "vae.data_std"):
|
||||
incomplete = {name: value for name, value in checkpoint.items() if name != key}
|
||||
with pytest.raises(RuntimeError, match=key):
|
||||
vae.load_state_dict(incomplete, strict=True)
|
||||
|
||||
latents = torch.randn(1, vae.vae.embed_dim, 4)
|
||||
mel_a = vae.decode(latents.clone())
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -23,10 +22,6 @@ from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
|
||||
refresh_context_on_transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend_context_manager,
|
||||
)
|
||||
@@ -45,22 +40,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils import precision
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
# Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which
|
||||
# need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's
|
||||
# fixture of the same name.
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29513",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
|
||||
|
||||
|
||||
def _randomize(module: torch.nn.Module, seed: int) -> None:
|
||||
@@ -70,29 +50,18 @@ def _randomize(module: torch.nn.Module, seed: int) -> None:
|
||||
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
|
||||
|
||||
|
||||
def test_time_embeddings_forward_under_bf16_linear_weights():
|
||||
# Minimal, pinpoint reproduction of the review comment: a bf16-weight
|
||||
# Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not
|
||||
# raise, and must return bf16 (the Linear weights' dtype).
|
||||
module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
|
||||
@pytest.mark.parametrize(
|
||||
"modulation", [False, True], ids=["time-embedding", "modulation"]
|
||||
)
|
||||
def test_bf16_projection_accepts_fp32_conditioning(modulation):
|
||||
module = (
|
||||
Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
|
||||
if modulation
|
||||
else Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
|
||||
)
|
||||
_randomize(module, seed=0)
|
||||
module = module.to(torch.bfloat16)
|
||||
|
||||
out = module(torch.tensor([0.0, 500.0, 999.0]))
|
||||
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out.float()).all()
|
||||
|
||||
|
||||
def test_modulation_forward_under_bf16_linear_weights():
|
||||
# Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the
|
||||
# identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time
|
||||
# embedding with bf16 weights, it must not raise either.
|
||||
module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
|
||||
module = module.to(torch.bfloat16)
|
||||
|
||||
out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to
|
||||
|
||||
inputs = torch.randn(2, 8) if modulation else torch.tensor([0.0, 500.0, 999.0])
|
||||
out = module.to(torch.bfloat16)(inputs)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out.float()).all()
|
||||
|
||||
@@ -297,7 +266,6 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_
|
||||
t_expand=torch.tensor([500.0], device="cuda"),
|
||||
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
sparse_params=None,
|
||||
visual_token_type_ids=None,
|
||||
)
|
||||
|
||||
@@ -360,71 +328,10 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_
|
||||
runner.reset()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason=(
|
||||
"the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real "
|
||||
"CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 "
|
||||
"numerics are only meaningful on real hardware anyway"
|
||||
),
|
||||
)
|
||||
def test_kandinsky6_dit_real_bf16_forward_does_not_crash():
|
||||
"""Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose
|
||||
weights are bf16 -- the exact configuration the SGLang PR review comment flagged
|
||||
as broken (a fp32 sinusoidal embedding feeding a bf16 Linear)."""
|
||||
device = torch.device("cuda")
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
arch_kwargs = _tiny_multimodal_arch_kwargs()
|
||||
config.update_model_arch(dict(arch_kwargs))
|
||||
dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval()
|
||||
_randomize(dit, seed=1)
|
||||
dit = dit.to(device=device, dtype=torch.bfloat16)
|
||||
|
||||
batch, duration, height, width = 1, 2, 2, 2
|
||||
text_len, audio_len = 3, 5
|
||||
hidden_states = torch.randn(
|
||||
batch, duration, height, width, 4, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
hidden_states_audio = torch.randn(
|
||||
batch, audio_len, 4, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
encoder_hidden_states = torch.randn(
|
||||
batch, text_len, 8, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device)
|
||||
# Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the
|
||||
# caller), same as every real sampling loop -- exactly the dtype that used to
|
||||
# mismatch the bf16 Linear.
|
||||
timestep = torch.tensor([500.0], device=device)
|
||||
visual_rope_pos = (
|
||||
torch.arange(duration, device=device),
|
||||
torch.arange(height, device=device),
|
||||
torch.arange(width, device=device),
|
||||
)
|
||||
text_rope_pos = torch.arange(text_len, device=device)
|
||||
|
||||
video_out, audio_out = dit(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_audio=hidden_states_audio,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
timestep=timestep,
|
||||
pooled_projections=pooled_projections,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
text_rope_pos=text_rope_pos,
|
||||
)
|
||||
|
||||
assert video_out.shape == (batch, duration, height, width, 4)
|
||||
assert audio_out.shape == (batch, audio_len, 4)
|
||||
assert video_out.dtype == torch.bfloat16
|
||||
assert audio_out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(video_out.float()).all()
|
||||
assert torch.isfinite(audio_out.float()).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("separate_cfg", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
|
||||
def test_bf16_joint_forward_and_cache_dit_lifetime(separate_cfg):
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4}
|
||||
config.update_model_arch(arch)
|
||||
@@ -445,18 +352,19 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
|
||||
outputs = []
|
||||
for step in range(3):
|
||||
for branch in range(2 if separate_cfg else 1):
|
||||
outputs.append(
|
||||
dit(
|
||||
**(
|
||||
inputs
|
||||
| {
|
||||
"pooled_projections": inputs["pooled_projections"]
|
||||
+ branch
|
||||
}
|
||||
),
|
||||
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
|
||||
with set_forward_context(current_timestep=step, attn_metadata=None):
|
||||
outputs.append(
|
||||
dit(
|
||||
**(
|
||||
inputs
|
||||
| {
|
||||
"pooled_projections": inputs["pooled_projections"]
|
||||
+ branch
|
||||
}
|
||||
),
|
||||
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
|
||||
)
|
||||
)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def assert_exact(actual, expected):
|
||||
@@ -465,6 +373,12 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
|
||||
torch.testing.assert_close(tensor, target, rtol=0, atol=0)
|
||||
|
||||
reference = run_steps()
|
||||
for video, audio in reference:
|
||||
assert video.shape == inputs["hidden_states"].shape
|
||||
assert audio.shape == inputs["hidden_states_audio"].shape
|
||||
for tensor in (video, audio):
|
||||
assert tensor.dtype == torch.bfloat16
|
||||
assert torch.isfinite(tensor).all()
|
||||
enable_cache_on_transformer(
|
||||
dit,
|
||||
CacheDitConfig(
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor
|
||||
must aspect-preserving-resize-then-centre-crop, matching the diffusers
|
||||
reference's ``encode_i2va_first_frame``
|
||||
(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height,
|
||||
src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to
|
||||
``(height, width)``), not stretch-resize the whole source image to the
|
||||
target box (which distorts its aspect ratio whenever the source doesn't
|
||||
already match the target aspect ratio).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
|
||||
Kandinsky6ImageEncodingStage,
|
||||
)
|
||||
|
||||
|
||||
def _gradient_image(width: int, height: int) -> PIL.Image.Image:
|
||||
"""A deterministic, per-pixel-distinguishable RGB image (not a flat
|
||||
color), so resize/crop geometry is actually verifiable pixel-for-pixel.
|
||||
"""
|
||||
xs = np.linspace(0, 255, width, dtype=np.uint8)
|
||||
ys = np.linspace(0, 255, height, dtype=np.uint8)
|
||||
grid = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
grid[..., 0] = xs[None, :]
|
||||
grid[..., 1] = ys[:, None]
|
||||
grid[..., 2] = 128
|
||||
return PIL.Image.fromarray(grid, mode="RGB")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_h,src_w,height,width",
|
||||
[
|
||||
(1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case)
|
||||
(1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case)
|
||||
(1024, 1536, 512, 768), # already 3:2 -- matching aspect
|
||||
],
|
||||
)
|
||||
def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width):
|
||||
# encode_i2va_first_frame: scale = min(src_h/height, src_w/width);
|
||||
# resize to (int(src_h/scale), int(src_w/scale)), then centre-crop.
|
||||
scale = min(src_h / height, src_w / width)
|
||||
expected = (int(src_h / scale), int(src_w / scale))
|
||||
assert (
|
||||
Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width)
|
||||
== expected
|
||||
)
|
||||
# The resize always covers the target box on both axes (never smaller).
|
||||
assert expected[0] >= height
|
||||
assert expected[1] >= width
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_w,src_h,min_mean_abs_diff",
|
||||
[
|
||||
(1920, 1080, 0.015), # 16:9 landscape source (audit's own case)
|
||||
(1080, 1920, 0.05), # 9:16 portrait source (audit's own case)
|
||||
],
|
||||
)
|
||||
def test_mismatched_aspect_pil_image_is_cropped_not_stretched(
|
||||
src_w, src_h, min_mean_abs_diff
|
||||
):
|
||||
# The model's documented default target resolution (512x768, 2:3)
|
||||
# against realistic phone/webcam source aspect ratios.
|
||||
height, width = 512, 768
|
||||
image = _gradient_image(src_w, src_h)
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
|
||||
assert produced.shape == (1, 3, height, width)
|
||||
|
||||
# Independent oracle: the diffusers reference's own aspect-preserving
|
||||
# resize + centre-crop, computed directly via PIL, without calling any
|
||||
# of the code under test.
|
||||
scale = min(src_h / height, src_w / width)
|
||||
new_h, new_w = int(src_h / scale), int(src_w / scale)
|
||||
resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
|
||||
top, left = (new_h - height) // 2, (new_w - width) // 2
|
||||
cropped = resized.crop((left, top, left + width, top + height))
|
||||
expected_np = np.array(cropped).astype(np.float32) / 255.0
|
||||
expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
|
||||
assert torch.allclose(produced[0], expected, atol=2.0 / 255.0)
|
||||
|
||||
# The deleted stretch-resize procedure (plain `.resize((width, height))`,
|
||||
# no crop) produces a materially different image whenever the source
|
||||
# aspect ratio doesn't already match the target -- proving this is a
|
||||
# real behavior change, not a no-op refactor.
|
||||
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
|
||||
stretched_np = np.array(stretched).astype(np.float32) / 255.0
|
||||
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
|
||||
assert mean_abs_diff > min_mean_abs_diff
|
||||
|
||||
|
||||
def test_matching_aspect_pil_image_resize_only_agrees_with_stretch():
|
||||
# A sanity check on the oracle itself: when the source already matches
|
||||
# the target aspect ratio, cover-resize-then-crop and plain stretch
|
||||
# resize agree closely (no crop actually removes anything) -- so the
|
||||
# large the divergence asserted above is specifically an aspect-ratio
|
||||
# effect, not a general property of the two code paths.
|
||||
height, width = 64, 96
|
||||
src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3...
|
||||
# (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2)
|
||||
image = _gradient_image(src_w, src_h)
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
|
||||
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
|
||||
stretched_np = np.array(stretched).astype(np.float32) / 255.0
|
||||
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
|
||||
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
|
||||
assert mean_abs_diff < 0.02
|
||||
|
||||
|
||||
def test_tensor_input_uses_cover_crop_geometry_too():
|
||||
height, width = 64, 96
|
||||
src_h, src_w = 108, 192
|
||||
tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1]
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width)
|
||||
assert produced.shape[-2:] == (height, width)
|
||||
@@ -19,10 +19,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend_context_manager,
|
||||
)
|
||||
@@ -47,24 +43,12 @@ from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import (
|
||||
TINY_KVAE,
|
||||
TINY_LU_MODEL,
|
||||
)
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def single_gpu(monkeypatch, default_global_server_args):
|
||||
if not model_parallel_is_initialized():
|
||||
port = get_free_port_below_ephemeral()
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
monkeypatch.setenv(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
def single_gpu(monkeypatch, default_global_server_args, single_process_model_parallel):
|
||||
monkeypatch.setattr(
|
||||
precision._mixed_precision_state,
|
||||
"state",
|
||||
|
||||
@@ -84,15 +84,6 @@ def _make_pipeline() -> Kandinsky6TI2VAPipeline:
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_pipeline_identity_and_required_modules():
|
||||
assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
|
||||
assert Kandinsky6TI2VAPipeline.is_video_pipeline is True
|
||||
assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
|
||||
# No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder
|
||||
# and the BigVGAN vocoder in one checkpoint component.
|
||||
assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules
|
||||
|
||||
|
||||
def test_disagg_role_rejects_non_monolithic_deployment():
|
||||
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
|
||||
|
||||
@@ -107,17 +98,16 @@ def test_disagg_role_rejects_non_monolithic_deployment():
|
||||
|
||||
def test_create_pipeline_stages_produces_documented_order():
|
||||
pipeline = _make_pipeline()
|
||||
server_args = MagicMock()
|
||||
|
||||
Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args)
|
||||
|
||||
actual_types = [type(stage) for stage in pipeline.stages]
|
||||
expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER]
|
||||
assert actual_types == expected_types
|
||||
|
||||
actual_names = list(pipeline._stage_name_mapping.keys())
|
||||
expected_names = [name for name, _ in EXPECTED_STAGE_ORDER]
|
||||
assert actual_names == expected_names
|
||||
assert pipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
|
||||
assert pipeline.is_video_pipeline is True
|
||||
assert pipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
|
||||
pipeline.create_pipeline_stages(MagicMock())
|
||||
assert [type(stage) for stage in pipeline.stages] == [
|
||||
stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER
|
||||
]
|
||||
assert [
|
||||
(name, type(stage)) for name, stage in pipeline._stage_name_mapping.items()
|
||||
] == EXPECTED_STAGE_ORDER
|
||||
|
||||
# get_module wiring: each stage received the mocked module it asked for.
|
||||
denoising_stage = pipeline._stage_name_mapping["denoising_stage"]
|
||||
@@ -224,7 +214,6 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
|
||||
t_expand=torch.zeros(1),
|
||||
visual_rope_pos=[],
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
sparse_params=None,
|
||||
visual_token_type_ids=None,
|
||||
)
|
||||
for result, reference in zip(actual, expected, strict=True):
|
||||
@@ -237,10 +226,7 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
|
||||
def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler(
|
||||
guidance_scale,
|
||||
):
|
||||
"""The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance,
|
||||
so the run must not silently ignore a guidance the caller asked for; the
|
||||
Diffusers pipeline raises the same way ("PiflowScheduler requires
|
||||
guidance_weight=1.0")."""
|
||||
"""The distilled checkpoint must reject unsupported CFG, not silently ignore it."""
|
||||
scheduler = _pro_distill_scheduler()
|
||||
batch = SimpleNamespace(
|
||||
timesteps=scheduler.timesteps,
|
||||
@@ -272,11 +258,6 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
|
||||
text_lens = []
|
||||
|
||||
class _FakeDiT(torch.nn.Module):
|
||||
"""A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage``
|
||||
now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks
|
||||
``self.transformer.modules()`` to infer the attention backend -- a plain
|
||||
callable has no ``.modules()``."""
|
||||
|
||||
def forward(self, **kwargs):
|
||||
assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0)
|
||||
text_lens.append(kwargs["encoder_hidden_states"].shape[1])
|
||||
@@ -309,28 +290,13 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
|
||||
is_warmup=True,
|
||||
sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"),
|
||||
)
|
||||
arch = SimpleNamespace(
|
||||
attention_engine="auto",
|
||||
in_visual_dim=video_channels,
|
||||
patch_size=(1, 2, 2),
|
||||
scale_factor=(1.0, 2.0, 2.0),
|
||||
)
|
||||
pipeline_config = SimpleNamespace(
|
||||
dit_config=SimpleNamespace(arch_config=arch),
|
||||
dit_precision="fp32",
|
||||
vae_config=SimpleNamespace(
|
||||
arch_config=SimpleNamespace(spatial_compression_ratio=8)
|
||||
),
|
||||
get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0],
|
||||
# CFG is disabled in this test (guidance_scale=1.0,
|
||||
# do_classifier_free_guidance=False), so the policy never needs a
|
||||
# negative branch, but Kandinsky6DenoisingStage still reads
|
||||
# `cfg_policy` / `get_classifier_free_guidance_scale` off the config
|
||||
# unconditionally to build the (single-branch) CFGPolicy.
|
||||
cfg_policy=CFGPolicy(),
|
||||
get_classifier_free_guidance_scale=lambda _batch, guidance_scale: (
|
||||
guidance_scale
|
||||
),
|
||||
pipeline_config = Kandinsky6TI2VAPipelineConfig(dit_precision="fp32")
|
||||
pipeline_config.dit_config.update_model_arch(
|
||||
dict(
|
||||
in_visual_dim=video_channels,
|
||||
patch_size=(1, 2, 2),
|
||||
scale_factor=(1.0, 2.0, 2.0),
|
||||
)
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=pipeline_config, enable_cfg_parallel=False
|
||||
|
||||
@@ -120,4 +120,4 @@ def test_gate_sum_preserves_fp32_intermediates():
|
||||
gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384]
|
||||
expected = (x.float() + gate.float() * update.float()).to(x.dtype)
|
||||
assert not torch.equal(expected, x + gate * update)
|
||||
assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected)
|
||||
assert torch.equal(kandinsky6.residual_gate_fp32(x, update, gate), expected)
|
||||
|
||||
@@ -5,6 +5,7 @@ import argparse
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from kandinsky6_sr_tiny_components import TINY_DIT
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRArchConfig,
|
||||
@@ -25,85 +26,10 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=[8, 4, 4],
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
attention_params={"512": {"type": "flash"}},
|
||||
use_text=False,
|
||||
)
|
||||
TINY_PIFLOW = dict(
|
||||
nfe=2,
|
||||
num_policy_substeps=8,
|
||||
final_step_size_scale=0.5,
|
||||
shift=5.0,
|
||||
n_grid=3,
|
||||
eps=1e-6,
|
||||
)
|
||||
SR_PARAMS = dict(
|
||||
scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
fps=24,
|
||||
)
|
||||
|
||||
|
||||
def flat_config(**changes):
|
||||
"""A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied."""
|
||||
flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3)
|
||||
flat.update(
|
||||
piflow_nfe=2,
|
||||
piflow_num_policy_substeps=8,
|
||||
piflow_final_step_size_scale=0.5,
|
||||
piflow_shift=5.0,
|
||||
piflow_eps=1e-6,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
sr_scheduler_scale=5.0,
|
||||
sr_lq_noise_scale=0.7,
|
||||
sr_lq_noise_type="ddpm",
|
||||
sr_lq_channel_noise_scale=0.0,
|
||||
sr_cap_noise_timestep=False,
|
||||
sr_fps=24,
|
||||
)
|
||||
flat.update(changes)
|
||||
return flat
|
||||
|
||||
|
||||
def official_config(**changes):
|
||||
"""An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head
|
||||
width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys.
|
||||
"""
|
||||
official = {
|
||||
key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",)
|
||||
}
|
||||
official.update(
|
||||
out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"],
|
||||
attribute_overrides=None,
|
||||
sr_params=dict(SR_PARAMS),
|
||||
)
|
||||
official.update(changes)
|
||||
return official
|
||||
|
||||
|
||||
def parse(flat):
|
||||
def parse(**changes):
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(flat)
|
||||
config.update_model_arch(dict(TINY_DIT, **changes))
|
||||
return config.arch_config
|
||||
|
||||
|
||||
@@ -111,8 +37,7 @@ def parse(flat):
|
||||
# DiT arch config
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
|
||||
arch = parse(official_config())
|
||||
assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler
|
||||
arch = parse()
|
||||
assert arch.out_visual_dim == 12 and arch.attribute_overrides == {}
|
||||
assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4)
|
||||
assert (arch.hidden_size, arch.num_attention_heads) == (32, 2)
|
||||
@@ -146,7 +71,7 @@ def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
|
||||
|
||||
def test_unknown_sr_params_keys_are_rejected():
|
||||
with pytest.raises(ValueError, match="sr_params has unsupported keys"):
|
||||
parse(official_config(sr_params={"fps": 24, "new_knob": 1}))
|
||||
parse(sr_params={"fps": 24, "new_knob": 1})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -159,6 +84,7 @@ def test_unknown_sr_params_keys_are_rejected():
|
||||
({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"),
|
||||
({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"),
|
||||
({"piflow_eps": None}, ValueError, "piflow_eps"),
|
||||
({"n_grid": 3}, ValueError, "n_grid"),
|
||||
({"patch_size": [2, 2, 2]}, ValueError, "patch_size"),
|
||||
({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"),
|
||||
],
|
||||
@@ -171,25 +97,28 @@ def test_unsupported_or_unknown_config_fails_loudly_naming_the_key(
|
||||
keys raise ValueError (``update_model_arch`` alone would park unknown keys in
|
||||
``extra_attrs``)."""
|
||||
with pytest.raises(error, match=flag):
|
||||
parse(flat_config(**changes))
|
||||
parse(**changes)
|
||||
|
||||
|
||||
def test_sparse_attention_request_is_detected_from_attention_params():
|
||||
assert parse(flat_config()).requested_sparse_attention() is None
|
||||
nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}})
|
||||
assert parse(nabla).requested_sparse_attention() == "nabla"
|
||||
# an override can switch it off again
|
||||
overridden = flat_config(
|
||||
assert parse().requested_sparse_attention() is None
|
||||
assert (
|
||||
parse(
|
||||
attention_params={"512": {"type": "nabla", "P": 0.9}}
|
||||
).requested_sparse_attention()
|
||||
== "nabla"
|
||||
)
|
||||
overridden = parse(
|
||||
attention_params={"512": {"type": "nabla"}},
|
||||
attribute_overrides={"attention_params": {"512": {"type": "flash"}}},
|
||||
)
|
||||
assert parse(overridden).requested_sparse_attention() is None
|
||||
assert overridden.requested_sparse_attention() is None
|
||||
|
||||
|
||||
def test_default_arch_config_is_valid_and_text_free():
|
||||
"""Registry / ``PipelineConfig.from_kwargs`` instantiate the default config."""
|
||||
arch = Kandinsky6SRArchConfig()
|
||||
assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1
|
||||
assert arch.use_text is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -37,13 +37,13 @@ from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import (
|
||||
_SpatialChunkedConv3d,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_DENOISED_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
decode_chunks,
|
||||
)
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Check SR prompt admission and request fields without running a GPU scheduler.
|
||||
"""Prompt admission and SR fields through the real JSON and multipart endpoints."""
|
||||
|
||||
JSON uses TestClient; multipart tests call the parsed endpoint directly. The
|
||||
full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
@@ -22,269 +16,80 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||
resolve_sampling_params_cls,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
|
||||
_build_video_sampling_params,
|
||||
_multipart_video_extras,
|
||||
)
|
||||
|
||||
_SR_EXTRA_FIELDS = frozenset(
|
||||
{
|
||||
"sr_resolution_scale",
|
||||
"sr_tiles_batch_size",
|
||||
"sr_tile_min_overlap",
|
||||
"sr_target_resolution",
|
||||
"sr_target_resize_mode",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _fake_server_args(**overrides) -> SimpleNamespace:
|
||||
base = dict(
|
||||
@pytest.mark.parametrize("multipart", [False, True], ids=["json", "multipart"])
|
||||
@pytest.mark.parametrize("prompt_required", [False, True], ids=["sr", "generic"])
|
||||
def test_prompt_admission_and_sr_fields(tmp_path, multipart, prompt_required):
|
||||
args = SimpleNamespace(
|
||||
pipeline_config=Kandinsky6SRPipelineConfig(),
|
||||
pipeline_class_name="Kandinsky6SRPipeline",
|
||||
backend="auto",
|
||||
model_id=None,
|
||||
model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
|
||||
model_path="kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers",
|
||||
served_model_name="kandinsky6-sr-test",
|
||||
input_save_path=os.path.expanduser(
|
||||
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads"
|
||||
),
|
||||
output_path=os.path.expanduser(
|
||||
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output"
|
||||
),
|
||||
input_save_path=str(tmp_path / "uploads"),
|
||||
output_path=str(tmp_path / "outputs"),
|
||||
attention_backend_config={},
|
||||
enable_trace=False,
|
||||
comfyui_mode=False,
|
||||
num_gpus=1,
|
||||
)
|
||||
base.update(overrides)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def _patched_global_server_args(server_args):
|
||||
return patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
|
||||
server_args,
|
||||
assert resolve_sampling_params_cls(args) is Kandinsky6SRSamplingParams
|
||||
params_cls = SamplingParams if prompt_required else Kandinsky6SRSamplingParams
|
||||
fields = dict(
|
||||
sr_resolution_scale=4,
|
||||
sr_tiles_batch_size=3,
|
||||
sr_tile_min_overlap=0.3,
|
||||
sr_target_resolution="hd",
|
||||
sr_target_resize_mode="fit",
|
||||
)
|
||||
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == fields.keys()
|
||||
assert SamplingParams.video_request_extra_fields() == frozenset()
|
||||
payload = dict(video_path="unused.mp4", **fields)
|
||||
request = (
|
||||
{"files": {key: (None, str(value)) for key, value in payload.items()}}
|
||||
if multipart
|
||||
else {"json": payload}
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The two declared pieces (SamplingParams-level)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_prompt_optional_defaults_false_and_sr_overrides_it():
|
||||
"""A generic pipeline keeps requiring ``prompt``; only the SR params opt out."""
|
||||
assert SamplingParams.prompt_optional is False
|
||||
assert Kandinsky6SRSamplingParams.prompt_optional is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sampling_params_cls, status",
|
||||
[(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)],
|
||||
)
|
||||
def test_json_prompt_requirement_matches_pipeline(
|
||||
tmp_path, sampling_params_cls, status
|
||||
):
|
||||
app = FastAPI()
|
||||
app.include_router(video_api.router)
|
||||
server_args = _fake_server_args(
|
||||
input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs")
|
||||
)
|
||||
dispatch = AsyncMock()
|
||||
with (
|
||||
_patched_global_server_args(server_args),
|
||||
patch.object(
|
||||
video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
|
||||
args,
|
||||
),
|
||||
patch.object(video_api, "resolve_sampling_params_cls", return_value=params_cls),
|
||||
patch.object(video_api, "_dispatch_job_async", dispatch),
|
||||
TestClient(app) as client,
|
||||
):
|
||||
response = client.post(
|
||||
"/v1/videos",
|
||||
json={"video_path": "unused.mp4", "sr_resolution_scale": 4},
|
||||
)
|
||||
assert response.status_code == status, response.text
|
||||
if status == 200:
|
||||
response = client.post("/v1/videos", **request)
|
||||
assert response.status_code == (400 if prompt_required else 200), response.text
|
||||
if prompt_required:
|
||||
dispatch.assert_not_called()
|
||||
else:
|
||||
assert response.json()["status"] == "queued"
|
||||
dispatch.assert_awaited_once()
|
||||
_, batch = dispatch.call_args.args
|
||||
assert batch.prompt == ""
|
||||
assert batch.sampling_params.sr_resolution_scale == 4
|
||||
else:
|
||||
dispatch.assert_not_called()
|
||||
assert isinstance(batch.sampling_params, Kandinsky6SRSamplingParams)
|
||||
assert batch.prompt == "" and batch.video_path == "unused.mp4"
|
||||
assert {key: vars(batch.sampling_params)[key] for key in fields} == fields
|
||||
|
||||
|
||||
def test_sr_declares_its_cli_flags_as_multipart_extra_fields():
|
||||
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS
|
||||
# The base default (what every other model still gets) stays empty.
|
||||
assert SamplingParams.video_request_extra_fields() == frozenset()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The real multipart-field allowlist (video_api._multipart_video_extras)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class():
|
||||
raw_form = {
|
||||
"video_path": "unused.mp4",
|
||||
"sr_resolution_scale": "4",
|
||||
"sr_tiles_batch_size": "3",
|
||||
"not_a_declared_field": "dropped",
|
||||
}
|
||||
sr_extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
@pytest.mark.parametrize("params_cls", [Kandinsky6SRSamplingParams, SamplingParams])
|
||||
def test_multipart_allowlist_drops_undeclared_fields(params_cls):
|
||||
extras = video_api._multipart_video_extras(
|
||||
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3", "undeclared": "1"},
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=Kandinsky6SRSamplingParams,
|
||||
sampling_params_cls=params_cls,
|
||||
)
|
||||
assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4"
|
||||
assert sr_extras["sr_tiles_batch_size"] == 3
|
||||
assert "not_a_declared_field" not in sr_extras
|
||||
|
||||
# A model without the override (the pre-fix behaviour) drops the sr_* fields too.
|
||||
base_extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=SamplingParams,
|
||||
assert extras == (
|
||||
{"sr_resolution_scale": 4, "sr_tiles_batch_size": 3}
|
||||
if params_cls is Kandinsky6SRSamplingParams
|
||||
else {}
|
||||
)
|
||||
assert "sr_resolution_scale" not in base_extras
|
||||
assert "sr_tiles_batch_size" not in base_extras
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The registry-backed class resolution the admission gate reads
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline():
|
||||
server_args = _fake_server_args()
|
||||
assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# End-to-end request parsing: a prompt-less request field set -> sampling params
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields():
|
||||
"""The real kwargs-building code (``_build_video_sampling_params``), given a
|
||||
request built the way the multipart branch builds one for a prompt-optional
|
||||
pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a
|
||||
real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error."""
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
VideoGenerationsRequest,
|
||||
)
|
||||
|
||||
server_args = _fake_server_args()
|
||||
raw_form = {
|
||||
"video_path": "unused.mp4",
|
||||
"sr_resolution_scale": "4",
|
||||
"sr_tiles_batch_size": "3",
|
||||
}
|
||||
extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=Kandinsky6SRSamplingParams,
|
||||
)
|
||||
req = VideoGenerationsRequest(
|
||||
prompt="", # what video_api.py substitutes for a missing prompt
|
||||
video_path=extras.pop("video_path"),
|
||||
**extras,
|
||||
)
|
||||
|
||||
with _patched_global_server_args(server_args):
|
||||
sampling_params = _build_video_sampling_params("gap1-request-id", req)
|
||||
|
||||
assert isinstance(sampling_params, Kandinsky6SRSamplingParams)
|
||||
assert sampling_params.prompt == ""
|
||||
assert sampling_params.sr_resolution_scale == 4
|
||||
assert sampling_params.sr_tiles_batch_size == 3
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The full endpoint, called directly (see module docstring for why not TestClient)
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _FakeMultipartRequest:
|
||||
"""Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly."""
|
||||
|
||||
def __init__(self, form: dict):
|
||||
self.headers = {"content-type": "multipart/form-data; boundary=x"}
|
||||
self._form = form
|
||||
|
||||
async def form(self):
|
||||
return self._form
|
||||
|
||||
|
||||
def test_multipart_sr_request_without_prompt_is_accepted():
|
||||
"""A multipart SR request with no ``prompt`` field must be admitted (no 400), the
|
||||
``sr_*`` fields must reach the queued request, and it must fail on the pre-fix
|
||||
code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would
|
||||
reject this before any of the rest of ``create_video`` ran."""
|
||||
server_args = _fake_server_args()
|
||||
request = _FakeMultipartRequest(
|
||||
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"}
|
||||
)
|
||||
dispatched: list[tuple] = []
|
||||
|
||||
async def _fake_dispatch(job_id, batch, **kwargs):
|
||||
dispatched.append((job_id, batch, kwargs))
|
||||
|
||||
async def run():
|
||||
with (
|
||||
_patched_global_server_args(server_args),
|
||||
patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch),
|
||||
):
|
||||
response = await video_api.create_video(
|
||||
request,
|
||||
prompt=None, # <-- the field under test: no prompt supplied
|
||||
enhance_prompt=None,
|
||||
task_type=None,
|
||||
x264_preset=None,
|
||||
perf_dump_path=None,
|
||||
input_reference=None,
|
||||
reference_url=None,
|
||||
video_reference=None,
|
||||
video_url=None,
|
||||
video_path="unused.mp4",
|
||||
model=None,
|
||||
n=1,
|
||||
num_outputs_per_prompt=None,
|
||||
seconds=None,
|
||||
size=None,
|
||||
fps=None,
|
||||
num_frames=None,
|
||||
seed=None,
|
||||
generator_device="cuda",
|
||||
negative_prompt=None,
|
||||
guidance_scale=None,
|
||||
guidance_scale_2=None,
|
||||
true_cfg_scale=None,
|
||||
num_inference_steps=None,
|
||||
max_sequence_length=None,
|
||||
flow_shift=None,
|
||||
enable_teacache=None,
|
||||
enable_frame_interpolation=None,
|
||||
frame_interpolation_exp=None,
|
||||
frame_interpolation_scale=None,
|
||||
frame_interpolation_model_path=None,
|
||||
enable_upscaling=None,
|
||||
upscaling_model_path=None,
|
||||
upscaling_scale=None,
|
||||
output_quality=None,
|
||||
output_compression=None,
|
||||
output_path=None,
|
||||
extra_params=None,
|
||||
extra_body=None,
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
return response
|
||||
|
||||
response = asyncio.run(run())
|
||||
|
||||
assert isinstance(response, VideoResponse)
|
||||
assert response.status == "queued"
|
||||
assert len(dispatched) == 1
|
||||
_, batch, _ = dispatched[0]
|
||||
assert batch.prompt == ""
|
||||
assert batch.sampling_params.sr_resolution_scale == 4
|
||||
assert batch.sampling_params.sr_tiles_batch_size == 3
|
||||
|
||||
@@ -3,11 +3,10 @@
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL
|
||||
from kandinsky6_sr_tiny_components import TINY_DIT, TINY_KVAE, TINY_LU_MODEL
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
@@ -19,10 +18,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders import (
|
||||
component_loader as component_loader_module,
|
||||
)
|
||||
@@ -34,6 +29,7 @@ from sglang.multimodal_gen.runtime.loader.fsdp_load import (
|
||||
load_model_from_full_model_state_dict,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
@@ -42,39 +38,7 @@ from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscale
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=[8, 4, 4],
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
use_text=False,
|
||||
n_grid=3,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29508",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -138,100 +102,48 @@ def _load_through_the_real_loader(checkpoint, **changes):
|
||||
return model
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt():
|
||||
"""Positive control for the failure tests below, and the meta-device path: RoPE
|
||||
tables and time-embedding frequencies are not in the checkpoint and must be rebuilt
|
||||
by ``post_load_weights`` so that a loaded model actually runs."""
|
||||
"""Meta-init must rebuild non-checkpoint RoPE and time-embedding buffers."""
|
||||
reference = _dit()
|
||||
checkpoint = _official_state_dict(reference)
|
||||
model = _load_through_the_real_loader(checkpoint)
|
||||
model.post_load_weights()
|
||||
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
|
||||
assert not model.time_embeddings.freqs.is_meta
|
||||
torch.testing.assert_close(model.pooled_bias, reference.pooled_bias)
|
||||
assert model.out_layer.out_layer.weight.shape[0] == 4 * TINY_DIT["out_visual_dim"]
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model.to(device)
|
||||
x = torch.randn(1, 2, 4, 4, 4, device=device)
|
||||
rope_pos = [torch.arange(2, device=device) for _ in range(3)]
|
||||
out = model(x, torch.tensor([500.0], device=device), rope_pos)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
out = model(x, torch.tensor([500.0], device=device), rope_pos)
|
||||
assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing",
|
||||
"key,unexpected,message",
|
||||
[
|
||||
"pooled_bias",
|
||||
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
|
||||
("pooled_bias", False, "was not loaded"),
|
||||
(
|
||||
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
|
||||
False,
|
||||
"was not loaded",
|
||||
),
|
||||
("text_embeddings.in_layer.weight", True, "unexpected keys"),
|
||||
],
|
||||
)
|
||||
def test_missing_checkpoint_tensor_fails_the_load(missing):
|
||||
"""The generic loader zero-fills missing parameters whose names contain ``bias`` (and
|
||||
would only warn about others). For this model every parameter is required, so a
|
||||
partial checkpoint must raise instead of yielding a silently different network."""
|
||||
def test_incomplete_or_unexpected_checkpoint_fails_the_load(key, unexpected, message):
|
||||
# even bias parameters must fail instead of the generic loader's zero-fill fallback
|
||||
checkpoint = _official_state_dict(_dit())
|
||||
del checkpoint[missing]
|
||||
with pytest.raises(ValueError, match="was not loaded"):
|
||||
if unexpected:
|
||||
checkpoint[key] = torch.zeros(2, 2)
|
||||
else:
|
||||
del checkpoint[key]
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_load_through_the_real_loader(checkpoint)
|
||||
|
||||
|
||||
def test_unexpected_checkpoint_tensor_fails_the_load():
|
||||
checkpoint = _official_state_dict(_dit())
|
||||
checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2)
|
||||
with pytest.raises(ValueError, match="unexpected keys"):
|
||||
_load_through_the_real_loader(checkpoint)
|
||||
|
||||
|
||||
def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys():
|
||||
"""An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives
|
||||
in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``.
|
||||
"""
|
||||
official = {
|
||||
key: value
|
||||
for key, value in TINY_DIT.items()
|
||||
if key not in ("n_grid", "attribute_overrides")
|
||||
}
|
||||
official.update(
|
||||
out_visual_dim=12,
|
||||
attribute_overrides=None,
|
||||
sr_params=dict(
|
||||
scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
fps=24,
|
||||
),
|
||||
)
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(official)
|
||||
with torch.device("meta"):
|
||||
model = Kandinsky6SRTransformer3DModel(config, official)
|
||||
# head width = prod(patch_size) * the TOTAL out_visual_dim
|
||||
assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12
|
||||
assert model.base_out_visual_dim == 4 and model.n_grid == 1
|
||||
|
||||
load_model_from_full_model_state_dict(
|
||||
model,
|
||||
model.preprocess_loaded_state_dict(
|
||||
iter(
|
||||
_official_state_dict(_dit()).items()
|
||||
) # same shapes as the legacy DX head
|
||||
),
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
strict=False,
|
||||
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
|
||||
)
|
||||
model.post_load_weights()
|
||||
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
|
||||
|
||||
|
||||
def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise():
|
||||
"""A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise``
|
||||
override would feed ``in`` channels. That mismatch must be reported, and the
|
||||
@@ -260,9 +172,6 @@ def test_rope_angle_tables_stay_fp32_when_the_module_is_cast():
|
||||
assert buffer.dtype == torch.float32, name
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fake server args for the component loaders
|
||||
# --------------------------------------------------------------------------- #
|
||||
class FakeServerArgs:
|
||||
def __init__(self):
|
||||
self.pipeline_config = Kandinsky6SRPipelineConfig()
|
||||
@@ -365,12 +274,21 @@ def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True):
|
||||
return directory
|
||||
|
||||
|
||||
def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
|
||||
directory = _tiny_lu_bundle(tmp_path)
|
||||
bank, _ = LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
@pytest.mark.parametrize("legacy,library", [(False, "diffusers"), (True, "kandinsky6")])
|
||||
def test_latent_upscaler_loading_and_default_scale_order(tmp_path, legacy, library):
|
||||
models = [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in (("4x", "2x") if legacy else ("2x", "4x"))
|
||||
]
|
||||
directory = _tiny_lu_bundle(tmp_path, models=models, include_scales=not legacy)
|
||||
loader = component_loader_module.ComponentLoader.for_component_type(
|
||||
"latent_upscaler", library
|
||||
)
|
||||
assert isinstance(loader, LatentUpscalerLoader)
|
||||
bank, _ = loader.load(str(directory), FakeServerArgs(), "latent_upscaler", library)
|
||||
assert bank.scales == (2, 4)
|
||||
assert bank._models[0] is bank.for_scale(2)
|
||||
assert bank._models[1] is bank.for_scale(4)
|
||||
assert not bank.training
|
||||
assert all(
|
||||
p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters()
|
||||
@@ -382,75 +300,35 @@ def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
|
||||
bank.upscale(z, scale=8)
|
||||
|
||||
|
||||
def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path):
|
||||
models = [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in ("4x", "2x")
|
||||
]
|
||||
directory = _tiny_lu_bundle(
|
||||
tmp_path,
|
||||
models=models,
|
||||
include_scales=False,
|
||||
)
|
||||
bank, _ = LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert bank.scales == (2, 4)
|
||||
assert bank._models[0] is bank.for_scale(2)
|
||||
assert bank._models[1] is bank.for_scale(4)
|
||||
|
||||
|
||||
def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library():
|
||||
"""The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...]."""
|
||||
for library in ("diffusers", "kandinsky6"):
|
||||
loader = component_loader_module.ComponentLoader.for_component_type(
|
||||
"latent_upscaler", library
|
||||
)
|
||||
assert isinstance(loader, LatentUpscalerLoader)
|
||||
def test_latent_upscaler_rejects_the_wrong_library():
|
||||
with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"):
|
||||
component_loader_module.ComponentLoader.for_component_type(
|
||||
"latent_upscaler", "transformers"
|
||||
)
|
||||
|
||||
|
||||
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path):
|
||||
"""``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the
|
||||
customized loader raises, which hides real errors. This component must re-raise,
|
||||
with the offending config key or tensor named."""
|
||||
@pytest.mark.parametrize("error_kind", ["Missing key", "bogus_field"])
|
||||
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path, error_kind):
|
||||
directory = _tiny_lu_bundle(tmp_path)
|
||||
weights_path = directory / "diffusion_pytorch_model.safetensors"
|
||||
weights = load_file(str(weights_path))
|
||||
weights.pop(next(iter(weights)))
|
||||
save_file(weights, str(weights_path))
|
||||
if error_kind == "Missing key":
|
||||
path = directory / "diffusion_pytorch_model.safetensors"
|
||||
weights = load_file(str(path))
|
||||
weights.pop(next(iter(weights)))
|
||||
save_file(weights, str(path))
|
||||
else:
|
||||
path = directory / "config.json"
|
||||
config = json.loads(path.read_text())
|
||||
config["models"][0]["model"]["bogus_field"] = 1
|
||||
path.write_text(json.dumps(config))
|
||||
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
|
||||
LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert "Missing key" in str(error.value.__cause__)
|
||||
|
||||
bad = copy.deepcopy(TINY_LU_MODEL)
|
||||
bad["bogus_field"] = 1
|
||||
directory = _tiny_lu_bundle(
|
||||
tmp_path / "bad",
|
||||
models=[{"target_scale": "2x", "model": TINY_LU_MODEL}],
|
||||
scales=(2,),
|
||||
)
|
||||
config_path = directory / "config.json"
|
||||
config = json.loads(config_path.read_text())
|
||||
config["models"][0]["model"] = bad
|
||||
config_path.write_text(json.dumps(config))
|
||||
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
|
||||
LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert "bogus_field" in str(error.value.__cause__)
|
||||
assert error_kind in str(error.value.__cause__)
|
||||
|
||||
|
||||
def test_motion_attention_is_not_a_supported_entry_key():
|
||||
"""The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` /
|
||||
commit 950a5edb) only implements the released checkpoint, which never trains
|
||||
natten/shifted-window motion attention: a config that asks for it must fail loudly
|
||||
at construction instead of silently building an unsupported backend."""
|
||||
"""Unsupported motion attention must fail instead of silently using dense attention."""
|
||||
with_motion = {
|
||||
**copy.deepcopy(TINY_LU_MODEL),
|
||||
"enable_x2_entry": False,
|
||||
|
||||
+44
-61
@@ -3,7 +3,6 @@
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections import Counter
|
||||
from types import SimpleNamespace
|
||||
from typing import NamedTuple
|
||||
@@ -32,10 +31,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
PipelineComponentLoader,
|
||||
)
|
||||
@@ -61,9 +56,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
build_sampling_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
PiflowParams,
|
||||
)
|
||||
|
||||
|
||||
class Bundle(NamedTuple):
|
||||
@@ -77,7 +69,6 @@ class Bundle(NamedTuple):
|
||||
head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width
|
||||
n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid``
|
||||
scheduler_scale: float # ``sr_params.scheduler_scale``
|
||||
calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5
|
||||
|
||||
|
||||
FLOW = Bundle(
|
||||
@@ -89,7 +80,6 @@ FLOW = Bundle(
|
||||
head_width=64,
|
||||
n_grid=1,
|
||||
scheduler_scale=5.0,
|
||||
calls_per_tile=5,
|
||||
)
|
||||
DISTILLED = Bundle(
|
||||
model_index=DISTILLED_MODEL_INDEX,
|
||||
@@ -100,26 +90,13 @@ DISTILLED = Bundle(
|
||||
head_width=640,
|
||||
n_grid=10,
|
||||
scheduler_scale=3.5,
|
||||
calls_per_tile=2,
|
||||
)
|
||||
both_repos = pytest.mark.parametrize(
|
||||
"bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29509",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
|
||||
|
||||
|
||||
def _prefixes(keys, depth=2):
|
||||
@@ -162,17 +139,18 @@ def test_real_transformer_config_builds_the_release_dit(bundle):
|
||||
# ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its
|
||||
# n_grid in the scheduler config
|
||||
assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792)
|
||||
assert model.base_out_visual_dim == 64 and model.n_grid == 1
|
||||
assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width
|
||||
assert bundle.n_grid * model.in_visual_dim == bundle.head_width
|
||||
assert model.instruct_type == "hybrid_anchor" and model.visual_cond
|
||||
assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input
|
||||
arch = config.arch_config
|
||||
assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]}
|
||||
assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow
|
||||
assert arch.sr_scheduler_scale == bundle.scheduler_scale
|
||||
|
||||
|
||||
@both_repos
|
||||
def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path):
|
||||
def test_loaded_scheduler_controls_sampling_and_rejects_a_mismatched_head(
|
||||
bundle, tmp_path
|
||||
):
|
||||
scheduler = _load_scheduler(bundle, tmp_path)
|
||||
# PiflowScheduler subclasses the flow-matching scheduler: compare exact types
|
||||
assert type(scheduler) is bundle.scheduler_cls
|
||||
@@ -180,43 +158,48 @@ def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_pa
|
||||
if bundle is DISTILLED:
|
||||
assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10)
|
||||
|
||||
|
||||
@both_repos
|
||||
def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path):
|
||||
"""The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever
|
||||
``num_inference_steps`` is), the flow-matching repo to flow-Euler
|
||||
(``num_inference_steps`` calls per tile directly -- this repo's own convention, not
|
||||
the upstream Diffusers pipeline's timestep-grid-point count).
|
||||
"""
|
||||
scheduler = _load_scheduler(bundle, tmp_path)
|
||||
arch = _dit_config(bundle).arch_config
|
||||
for num_steps in (3, 5, 9):
|
||||
spec = _sampling_spec(arch, scheduler, num_steps)
|
||||
assert spec.is_piflow == (bundle is DISTILLED)
|
||||
assert spec.num_steps == (2 if bundle is DISTILLED else num_steps)
|
||||
mismatched = _dit_config(FLOW if bundle is DISTILLED else DISTILLED).arch_config
|
||||
with pytest.raises(ValueError, match="head.*same checkpoint"):
|
||||
_sampling_spec(mismatched, scheduler)
|
||||
|
||||
def spec(num_steps):
|
||||
return build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=num_steps,
|
||||
tiles_batch_size=1,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
|
||||
def _sampling_spec(arch, scheduler, num_steps=5):
|
||||
return build_sampling_spec(
|
||||
arch=arch,
|
||||
scheduler=scheduler,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=num_steps,
|
||||
tiles_batch_size=1,
|
||||
tile_min_overlap=0.2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nfe", [None, 0, -1])
|
||||
def test_piflow_requires_a_positive_checkpoint_step_count(nfe):
|
||||
with pytest.raises(ValueError, match="nfe >= 1"):
|
||||
_sampling_spec(
|
||||
_dit_config(DISTILLED).arch_config, PiflowScheduler(nfe=nfe, n_grid=10)
|
||||
)
|
||||
|
||||
assert spec(5).steps_per_chunk == bundle.calls_per_tile
|
||||
assert spec(5).scheduler_scale == bundle.scheduler_scale
|
||||
if bundle is DISTILLED:
|
||||
assert spec(5).piflow == PiflowParams(
|
||||
nfe=2,
|
||||
num_policy_substeps=128,
|
||||
final_step_size_scale=0.5,
|
||||
shift=3.5,
|
||||
n_grid=10,
|
||||
eps=1e-6,
|
||||
)
|
||||
assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2
|
||||
else:
|
||||
assert spec(5).piflow is None
|
||||
assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9)
|
||||
|
||||
def test_sampling_rejects_missing_scheduler_steps_and_unsupported_cap():
|
||||
arch = _dit_config(DISTILLED).arch_config
|
||||
for scheduler in (None, object()):
|
||||
with pytest.raises(ValueError, match="scheduler|Scheduler"):
|
||||
_sampling_spec(arch, scheduler)
|
||||
scheduler = PiflowScheduler(nfe=2, n_grid=10)
|
||||
with pytest.raises(ValueError, match="num_inference_steps"):
|
||||
_sampling_spec(arch, scheduler, num_steps=0)
|
||||
arch.sr_cap_noise_timestep = True
|
||||
arch.attribute_overrides = {"instruct_type": "noise"}
|
||||
with pytest.raises(NotImplementedError, match="cap_noise_timestep"):
|
||||
_sampling_spec(arch, scheduler)
|
||||
|
||||
|
||||
def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys():
|
||||
|
||||
@@ -23,7 +23,12 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve
|
||||
from kandinsky6_sr_tiny_components import (
|
||||
TINY_KVAE,
|
||||
TINY_LU_MODEL,
|
||||
TINY_PIFLOW,
|
||||
super_resolve,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
@@ -31,14 +36,11 @@ from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
get_param_names_mapping,
|
||||
hf_to_custom_state_dict,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
@@ -58,12 +60,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
denoise_with_scheduler,
|
||||
euler_start_timestep,
|
||||
make_dit_fn,
|
||||
SamplingSpec,
|
||||
denoise_chunks,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
@@ -105,14 +105,6 @@ TINY_DIT_CFG = dict(
|
||||
attention_params={"512": {"type": "flash"}},
|
||||
use_text=False,
|
||||
)
|
||||
TINY_PIFLOW = dict(
|
||||
nfe=2,
|
||||
num_policy_substeps=8,
|
||||
final_step_size_scale=0.5,
|
||||
shift=5.0,
|
||||
n_grid=3,
|
||||
eps=1e-6,
|
||||
)
|
||||
TINY_SR_PARAMS = dict(
|
||||
scale_factor={512: [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
@@ -203,20 +195,7 @@ from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402
|
||||
)
|
||||
from omegaconf import OmegaConf # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29507",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
|
||||
|
||||
|
||||
def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None:
|
||||
@@ -240,40 +219,16 @@ def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None
|
||||
return dit
|
||||
|
||||
|
||||
def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict:
|
||||
"""The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields)."""
|
||||
flat = dict(cfg)
|
||||
flat["patch_size"] = list(flat["patch_size"])
|
||||
flat["axes_dims"] = list(flat["axes_dims"])
|
||||
flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1
|
||||
if piflow is not None:
|
||||
flat.update(
|
||||
piflow_nfe=piflow["nfe"],
|
||||
piflow_num_policy_substeps=piflow["num_policy_substeps"],
|
||||
piflow_final_step_size_scale=piflow["final_step_size_scale"],
|
||||
piflow_shift=piflow["shift"],
|
||||
piflow_eps=piflow["eps"],
|
||||
)
|
||||
flat["attribute_overrides"] = dict(overrides)
|
||||
flat.update(
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"],
|
||||
sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"],
|
||||
sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"],
|
||||
sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"],
|
||||
sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"],
|
||||
sr_fps=TINY_SR_PARAMS["fps"],
|
||||
)
|
||||
return flat
|
||||
|
||||
|
||||
def build_port_dit(
|
||||
reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict
|
||||
) -> Kandinsky6SRTransformer3DModel:
|
||||
"""Port DiT loaded with the reference weights (``model.`` prefix), strictly."""
|
||||
flat = flat_transformer_config(
|
||||
cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides
|
||||
cfg = dict(cfg or TINY_DIT_CFG)
|
||||
flat = dict(
|
||||
cfg,
|
||||
out_visual_dim=cfg["out_visual_dim"] * (piflow["n_grid"] if piflow else 1),
|
||||
attribute_overrides=dict(overrides),
|
||||
sr_params=dict(TINY_SR_PARAMS),
|
||||
)
|
||||
dit_config = Kandinsky6SRDitConfig()
|
||||
dit_config.update_model_arch(flat)
|
||||
@@ -324,13 +279,14 @@ def _port_forward(model, x, time, *, scale_factor, motion_score=None):
|
||||
torch.arange(height // model.patch_size[1]),
|
||||
torch.arange(width // model.patch_size[2]),
|
||||
]
|
||||
return model(
|
||||
x,
|
||||
time,
|
||||
rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion_score,
|
||||
)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
return model(
|
||||
x,
|
||||
time,
|
||||
rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion_score,
|
||||
)
|
||||
|
||||
|
||||
def _random_latent(
|
||||
@@ -427,28 +383,36 @@ def _reference_loop_inputs(x):
|
||||
return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long)
|
||||
|
||||
|
||||
def _port_dit_fn(model, x, scale_factor, use_motion_score=False):
|
||||
return make_dit_fn(
|
||||
model,
|
||||
latent_frames_hw=tuple(x.shape[1:4]),
|
||||
patch_size=model.patch_size,
|
||||
scale_factor=scale_factor,
|
||||
use_motion_score=use_motion_score,
|
||||
def _port_sample(model, x, scale, scheduler, steps, capped=False):
|
||||
spec = SamplingSpec(
|
||||
tiling_scale=2,
|
||||
tiles_batch_size=1,
|
||||
seed=42,
|
||||
num_steps=steps,
|
||||
is_piflow=isinstance(scheduler, PiflowScheduler),
|
||||
tile_min_overlap=0.2,
|
||||
visual_size=512,
|
||||
scale_factor=scale,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=capped,
|
||||
)
|
||||
return denoise_chunks(
|
||||
[x.clone()],
|
||||
model,
|
||||
scheduler,
|
||||
dit_spec=build_dit_spec(model),
|
||||
spec=spec,
|
||||
device=x.device,
|
||||
step_context=lambda step: set_forward_context(
|
||||
current_timestep=step, attn_metadata=None
|
||||
),
|
||||
)[0]
|
||||
|
||||
|
||||
def test_piflow_loop_matches_reference_piflow_generate():
|
||||
"""Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles.
|
||||
|
||||
The reference packs the tiles along time with one per-frame sigma; the port batches
|
||||
``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU.
|
||||
|
||||
The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule``
|
||||
/ ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through
|
||||
``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the
|
||||
``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same
|
||||
per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did.
|
||||
"""
|
||||
"""Compare the packed reference loop with the batched native PiflowScheduler."""
|
||||
from kandinsky_sr.core.algo.piflow_sampler import piflow_generate
|
||||
|
||||
reference = build_reference_dit(piflow=TINY_PIFLOW)
|
||||
@@ -460,7 +424,6 @@ def test_piflow_loop_matches_reference_piflow_generate():
|
||||
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
|
||||
packed = x.reshape(-1, *x.shape[2:]).clone()
|
||||
scheduler = PiflowScheduler(**TINY_PIFLOW)
|
||||
scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu")
|
||||
with torch.no_grad():
|
||||
expected = piflow_generate(
|
||||
packed,
|
||||
@@ -476,13 +439,7 @@ def test_piflow_loop_matches_reference_piflow_generate():
|
||||
device="cpu",
|
||||
**TINY_PIFLOW,
|
||||
)
|
||||
actual = denoise_with_scheduler(
|
||||
x.clone(),
|
||||
_port_dit_fn(model, x, scale),
|
||||
scheduler,
|
||||
channels=4,
|
||||
is_piflow=True,
|
||||
)
|
||||
actual = _port_sample(model, x, scale, scheduler, TINY_PIFLOW["nfe"])
|
||||
torch.testing.assert_close(
|
||||
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
|
||||
)
|
||||
@@ -492,18 +449,8 @@ def test_piflow_loop_matches_reference_piflow_generate():
|
||||
"capped", [False, True], ids=["full_range", "capped_noise_start"]
|
||||
)
|
||||
def test_euler_loop_matches_reference_generate(capped):
|
||||
"""Guards the warped timesteps, per-step deltas and (for a wide input) the in-place
|
||||
update of the first channels only, incl. the ``cap_noise_timestep`` start time.
|
||||
|
||||
The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4
|
||||
Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent
|
||||
port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace
|
||||
array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift``
|
||||
warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself --
|
||||
algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4
|
||||
step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for-
|
||||
bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array.
|
||||
"""
|
||||
"""Compare noise caps and channel updates with the reference's five-point grid
|
||||
(four DiT calls); both schedulers append the same terminal zero."""
|
||||
from kandinsky_sr.core.algo.utils import generate
|
||||
|
||||
cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor")
|
||||
@@ -517,16 +464,11 @@ def test_euler_loop_matches_reference_generate(capped):
|
||||
reference.instruct_type, reference.visual_cond = "noise", True
|
||||
x = _random_latent(2 * 4 + 1, seed=9)
|
||||
scale = (1.0, 1.5, 1.5)
|
||||
start = euler_start_timestep(
|
||||
cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise"
|
||||
)
|
||||
assert start == (0.7 if capped else 1.0)
|
||||
start = 0.7 if capped else 1.0
|
||||
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
|
||||
packed = x.reshape(-1, *x.shape[2:]).clone()
|
||||
num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps
|
||||
scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0)
|
||||
sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist()
|
||||
scheduler.set_timesteps(sigmas=sigmas, device="cpu")
|
||||
with torch.no_grad():
|
||||
expected = generate(
|
||||
packed,
|
||||
@@ -546,13 +488,7 @@ def test_euler_loop_matches_reference_generate(capped):
|
||||
5.0,
|
||||
start_timestep=start,
|
||||
)
|
||||
actual = denoise_with_scheduler(
|
||||
x.clone(),
|
||||
_port_dit_fn(model, x, scale, use_motion_score=True),
|
||||
scheduler,
|
||||
channels=4,
|
||||
is_piflow=False,
|
||||
)
|
||||
actual = _port_sample(model, x, scale, scheduler, num_inference_steps, capped)
|
||||
torch.testing.assert_close(
|
||||
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
|
||||
)
|
||||
@@ -663,6 +599,9 @@ def build_stacks(*, piflow, use_lu):
|
||||
vae.load_state_dict(dict(ref_vae.state_dict()), strict=True)
|
||||
port = {
|
||||
"vae": vae.eval(),
|
||||
"scheduler": PiflowScheduler(**piflow)
|
||||
if piflow
|
||||
else FlowMatchEulerDiscreteScheduler(shift=TINY_SR_PARAMS["scheduler_scale"]),
|
||||
"dit": build_port_dit(
|
||||
ref_dit, piflow=piflow, overrides={"instruct_type": "noise"}
|
||||
),
|
||||
@@ -728,7 +667,10 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
|
||||
if pre != 1.0:
|
||||
video = tiling.pre_upscale_video(video, pre, 16)
|
||||
arch = port["dit"].config
|
||||
port_num_steps = num_steps if arch.is_piflow else num_steps - 1
|
||||
scheduler = port["scheduler"]
|
||||
port_num_steps = (
|
||||
num_steps if isinstance(scheduler, PiflowScheduler) else num_steps - 1
|
||||
)
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=tiling_scale,
|
||||
@@ -736,6 +678,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
|
||||
num_steps=port_num_steps,
|
||||
tiles_batch_size=tiles_batch_size,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
bank = port["bank"]
|
||||
use_lu = bank is not None and bank.for_scale(tiling_scale) is not None
|
||||
@@ -746,7 +689,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
|
||||
dit=port["dit"],
|
||||
dit_spec=build_dit_spec(port["dit"]),
|
||||
spec=spec,
|
||||
scheduler=effective_scheduler(spec, None),
|
||||
scheduler=scheduler,
|
||||
device="cpu",
|
||||
upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None,
|
||||
lu_dtype=module_dtype(bank) if use_lu else None,
|
||||
@@ -782,19 +725,10 @@ def _assert_uint8_close(actual, expected, *, max_diff: int = 1):
|
||||
def test_tiled_piflow_super_resolution_matches_reference(
|
||||
use_lu, scale, hw, tiles_batch_size
|
||||
):
|
||||
"""Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or
|
||||
tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with
|
||||
truncation to uint8 and Hann stitching, against ``run_tiled_sr`` /
|
||||
``run_tiled_sr_from_pixels`` with identical random weights.
|
||||
"""Compare tiled pixel/LU paths with identical random weights and seeds.
|
||||
|
||||
``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the
|
||||
training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to
|
||||
0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``).
|
||||
The port now deliberately matches the real ``PiflowScheduler._policy_step``
|
||||
convention (``raw_dst=eps``) instead, which real checkpoints always run through.
|
||||
That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up
|
||||
here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing
|
||||
rounding-boundary noise (still >99.99% exact pixels), not as a structural bug.
|
||||
The reference ends at raw_dst=0, whereas PiflowScheduler ends at eps (1e-6).
|
||||
Allow two uint8 levels of error while requiring more than 99.9% exact pixels.
|
||||
"""
|
||||
reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu)
|
||||
video = _random_video(9, *hw)
|
||||
|
||||
@@ -1,253 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Validate scheduler selection, DiT call counts and incompatible head rejection."""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from kandinsky6_sr_release_configs import (
|
||||
DISTILLED_SCHEDULER_CONFIG,
|
||||
DISTILLED_TRANSFORMER_CONFIG,
|
||||
FLOW_SCHEDULER_CONFIG,
|
||||
FLOW_TRANSFORMER_CONFIG,
|
||||
)
|
||||
from kandinsky6_sr_tiny_components import (
|
||||
TINY_DIT,
|
||||
build_components,
|
||||
make_request,
|
||||
make_stage,
|
||||
random_video,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_VIDEO_KEY,
|
||||
build_sampling_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
)
|
||||
|
||||
LATENT_WIDTH = TINY_DIT["in_visual_dim"]
|
||||
N_GRID = (
|
||||
3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields
|
||||
)
|
||||
NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles
|
||||
|
||||
|
||||
def official_dit(head_width):
|
||||
"""Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL
|
||||
head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config."""
|
||||
arch = {
|
||||
key: value
|
||||
for key, value in TINY_DIT.items()
|
||||
if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_"))
|
||||
}
|
||||
return dict(
|
||||
arch,
|
||||
out_visual_dim=head_width,
|
||||
sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]),
|
||||
)
|
||||
|
||||
|
||||
FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head
|
||||
DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head
|
||||
|
||||
|
||||
def flow_scheduler():
|
||||
return FlowMatchEulerDiscreteScheduler(shift=5.0)
|
||||
|
||||
|
||||
def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID):
|
||||
return PiflowScheduler(
|
||||
nfe=nfe,
|
||||
n_grid=n_grid,
|
||||
shift=5.0,
|
||||
eps=1e-6,
|
||||
final_step_size_scale=0.5,
|
||||
num_policy_substeps=8,
|
||||
)
|
||||
|
||||
|
||||
def release_scheduler(scheduler_cls, config):
|
||||
return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")})
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29510",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def tiny_resolutions(monkeypatch):
|
||||
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
|
||||
|
||||
|
||||
class SamplingRun(NamedTuple):
|
||||
tiles: int
|
||||
dit_calls: int
|
||||
progress_totals: list
|
||||
|
||||
|
||||
def run_denoising(dit_arch, scheduler, *, num_steps=5):
|
||||
"""Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count
|
||||
the DiT calls next to the totals the denoising progress bar was opened with."""
|
||||
vae, dit, _, server_args = build_components((), dit_arch)
|
||||
dit_calls = []
|
||||
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
|
||||
latent_prep = make_stage(
|
||||
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler
|
||||
)
|
||||
denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler)
|
||||
progress_totals = []
|
||||
progress_bar = denoise.progress_bar
|
||||
|
||||
def spy_on_progress_bar(**kwargs):
|
||||
progress_totals.append(kwargs["total"])
|
||||
return progress_bar(**kwargs)
|
||||
|
||||
denoise.progress_bar = spy_on_progress_bar
|
||||
|
||||
batch = make_request(random_video(9, 64, 96), num_steps=num_steps)
|
||||
batch = latent_prep.forward(batch, server_args)
|
||||
tiles = len(batch.extra["kandinsky6_sr_chunks"])
|
||||
denoise.forward(batch, server_args)
|
||||
return SamplingRun(tiles, len(dit_calls), progress_totals)
|
||||
|
||||
|
||||
def sampling_spec(dit_arch, scheduler):
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(dict(dit_arch))
|
||||
return build_sampling_spec(
|
||||
arch=config.arch_config,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=5,
|
||||
tiles_batch_size=1,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT calls per tile
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)])
|
||||
def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps(
|
||||
nfe, num_steps
|
||||
):
|
||||
run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps)
|
||||
assert run.tiles == NUM_TILES
|
||||
assert run.dit_calls == NUM_TILES * nfe
|
||||
assert run.progress_totals == [NUM_TILES * nfe]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)])
|
||||
def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile(
|
||||
num_steps, calls_per_tile
|
||||
):
|
||||
run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps)
|
||||
assert run.tiles == NUM_TILES
|
||||
assert run.dit_calls == NUM_TILES * calls_per_tile
|
||||
assert run.progress_totals == [NUM_TILES * calls_per_tile]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_steps", [3, 9])
|
||||
def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps):
|
||||
"""A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the
|
||||
denoising stage synthesizes a matching PiflowScheduler from those arch fields
|
||||
(run_spec.effective_scheduler) instead of leaving nothing to step."""
|
||||
run = run_denoising(TINY_DIT, None, num_steps=num_steps)
|
||||
assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"]
|
||||
assert run.progress_totals == [run.dit_calls]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Samplers that do not fit the DiT head
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize(
|
||||
"scheduler",
|
||||
[flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None],
|
||||
ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"],
|
||||
)
|
||||
def test_dx_head_never_runs_flow_euler(scheduler):
|
||||
with pytest.raises(ValueError, match="flow-Euler"):
|
||||
sampling_spec(DX_DIT, scheduler)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dit_arch,n_grid,needed",
|
||||
[
|
||||
(FLOW_DIT, N_GRID, "3 x 4 = 12"),
|
||||
(DX_DIT, 2, "2 x 4 = 8"),
|
||||
(DX_DIT, 4, "4 x 4 = 16"),
|
||||
],
|
||||
ids=["single_head", "too_few_grids", "too_many_grids"],
|
||||
)
|
||||
def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed):
|
||||
with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"):
|
||||
sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nfe", [0, -1])
|
||||
def test_pi_flow_needs_at_least_one_call_per_tile(nfe):
|
||||
with pytest.raises(ValueError, match="nfe >= 1"):
|
||||
sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe))
|
||||
with pytest.raises(ValueError, match="nfe >= 1"):
|
||||
sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None)
|
||||
|
||||
|
||||
def test_release_transformer_and_scheduler_of_different_repos_are_rejected():
|
||||
"""The real configs of the two official repos: their transformers differ only in
|
||||
the head width, so one repo's transformer with the other's scheduler must fail."""
|
||||
flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG)
|
||||
distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG)
|
||||
|
||||
assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5
|
||||
assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2
|
||||
with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"):
|
||||
sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow)
|
||||
with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"):
|
||||
sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled)
|
||||
|
||||
|
||||
def test_a_mismatched_bundle_fails_before_any_tile_is_processed():
|
||||
vae, dit, _, server_args = build_components((), DX_DIT)
|
||||
dit_calls = []
|
||||
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
|
||||
latent_prep = make_stage(
|
||||
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler()
|
||||
)
|
||||
batch = make_request(random_video(9, 64, 96))
|
||||
|
||||
with pytest.raises(ValueError, match="flow-Euler"):
|
||||
latent_prep.forward(batch, server_args)
|
||||
assert dit_calls == []
|
||||
assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles
|
||||
@@ -2,14 +2,15 @@
|
||||
"""Native SR stage-chain parity, component residency and video/audio hand-offs."""
|
||||
|
||||
import copy
|
||||
import os
|
||||
from functools import partial
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import (
|
||||
TINY_DIT,
|
||||
TINY_LU_MODEL,
|
||||
TINY_PIFLOW,
|
||||
build_components,
|
||||
make_request,
|
||||
make_stage,
|
||||
@@ -21,9 +22,11 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
@@ -32,26 +35,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
|
||||
Kandinsky6SRPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
|
||||
Kandinsky6SREncodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
WARMUP_CLIP_FRAMES,
|
||||
WARMUP_CLIP_HW,
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
|
||||
Kandinsky6SROutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_PLAN_KEY,
|
||||
@@ -60,11 +43,20 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
SR_VIDEO_KEY,
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
|
||||
WARMUP_CLIP_FRAMES,
|
||||
WARMUP_CLIP_HW,
|
||||
Kandinsky6SRDecodeStage,
|
||||
Kandinsky6SRDenoisingStage,
|
||||
Kandinsky6SREncodeStage,
|
||||
Kandinsky6SRInputStage,
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
Kandinsky6SROutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
TilePlan,
|
||||
)
|
||||
@@ -72,19 +64,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
RESOLUTIONS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29509",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -93,22 +73,25 @@ def tiny_resolutions(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bank_scales",
|
||||
[("2x", "4x"), (), ("4x",)],
|
||||
"bank_scales,nfe,num_steps",
|
||||
[(("2x", "4x"), 2, 5), ((), None, 3), (("4x",), 3, 9)],
|
||||
ids=["lu_path", "pixel_path", "pixel_fallback"],
|
||||
)
|
||||
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
"""encode -> latent-prep -> denoise -> decode -> output must give what
|
||||
``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank
|
||||
that has no entry for the requested scale (which must fall back to the pixel path, not
|
||||
fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and
|
||||
2."""
|
||||
vae, dit, bank, server_args = build_components(bank_scales)
|
||||
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales, nfe, num_steps):
|
||||
"""Cover LU/pixel fallback and scheduler reuse across chunks and requests."""
|
||||
scheduler = (
|
||||
PiflowScheduler(**dict(TINY_PIFLOW, nfe=nfe))
|
||||
if nfe is not None
|
||||
else FlowMatchEulerDiscreteScheduler(shift=5.0)
|
||||
)
|
||||
vae, dit, bank, server_args = build_components(
|
||||
bank_scales, dict(TINY_DIT, out_visual_dim=12 if nfe is not None else 4)
|
||||
)
|
||||
video = random_video(9, 64, 128)
|
||||
batch = make_request(video, tiles_batch_size=5)
|
||||
batch = make_request(video, tiles_batch_size=5, num_steps=num_steps)
|
||||
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None)
|
||||
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=scheduler)
|
||||
if bank is not None:
|
||||
pipeline.modules["latent_upscaler"] = bank
|
||||
pipeline._stages, pipeline._stage_name_mapping = [], {}
|
||||
@@ -148,7 +131,14 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra
|
||||
assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk
|
||||
assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan)
|
||||
batch = denoise.forward(batch, server_args)
|
||||
dit_calls = []
|
||||
hook = dit.register_forward_hook(lambda *args: dit_calls.append(1))
|
||||
with patch.object(denoise, "progress_bar", wraps=denoise.progress_bar) as progress:
|
||||
batch = denoise.forward(batch, server_args)
|
||||
hook.remove()
|
||||
assert len(dit_calls) == 3 * (nfe or num_steps)
|
||||
assert progress.call_args.kwargs["total"] == len(dit_calls)
|
||||
assert denoise.scheduler is scheduler
|
||||
batch = decode.forward(batch, server_args)
|
||||
assert len(batch.extra[SR_TILES_KEY]) == 12
|
||||
result = output.forward(batch, server_args)
|
||||
@@ -159,9 +149,10 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
arch=arch,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=5,
|
||||
num_steps=num_steps,
|
||||
tiles_batch_size=5,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
expected = super_resolve(
|
||||
video,
|
||||
@@ -170,7 +161,7 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
dit=dit,
|
||||
dit_spec=build_dit_spec(dit),
|
||||
spec=spec,
|
||||
scheduler=effective_scheduler(spec, None),
|
||||
scheduler=scheduler,
|
||||
device=next(dit.parameters()).device,
|
||||
upscale_fn=partial(bank.upscale, scale=2) if use_lu else None,
|
||||
lu_dtype=module_dtype(bank) if use_lu else None,
|
||||
@@ -184,6 +175,24 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra
|
||||
|
||||
|
||||
def test_mismatched_sampler_fails_before_encoding_tiles():
|
||||
vae, dit, _, server_args = build_components(())
|
||||
stage = make_stage(
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
server_args,
|
||||
vae,
|
||||
dit,
|
||||
None,
|
||||
FlowMatchEulerDiscreteScheduler(shift=5.0),
|
||||
)
|
||||
batch = make_request(random_video(9, 64, 96))
|
||||
with patch.object(vae, "encode", wraps=vae.encode) as encode:
|
||||
with pytest.raises(ValueError, match="head.*same checkpoint"):
|
||||
stage.forward(batch, server_args)
|
||||
encode.assert_not_called()
|
||||
assert SR_VIDEO_KEY in batch.extra
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER]
|
||||
)
|
||||
|
||||
@@ -9,7 +9,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
|
||||
SamplingSpec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
build_chunk_latent,
|
||||
plan_tiles,
|
||||
prepare_tile_latents,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
compute_tile_grid_even,
|
||||
@@ -63,22 +64,30 @@ def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype):
|
||||
tiles_batch_size=1,
|
||||
seed=1,
|
||||
num_steps=5,
|
||||
is_piflow=False,
|
||||
tile_min_overlap=0.2,
|
||||
visual_size=512,
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="linear",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
piflow=None,
|
||||
)
|
||||
tile = torch.randn(4, 4, 4, 4, dtype=torch.float64)
|
||||
result = build_chunk_latent(
|
||||
[tile],
|
||||
seed=1,
|
||||
plan = plan_tiles(
|
||||
frame_hw=(64, 64),
|
||||
visual_size=512,
|
||||
tiling_scale=2,
|
||||
tile_min_overlap=0.2,
|
||||
resolutions={512: [(128, 128)]},
|
||||
)
|
||||
result = prepare_tile_latents(
|
||||
tile,
|
||||
plan,
|
||||
tile_encoder=lambda tile: tile.permute(0, 2, 3, 1),
|
||||
latent_path=True,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
)[0]
|
||||
assert result.dtype == (dit_dtype or tile.dtype)
|
||||
|
||||
@@ -15,28 +15,39 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
FrameSelector,
|
||||
decode_clip,
|
||||
extract_source_audio,
|
||||
to_output_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
clip_to_aligned_frames,
|
||||
resample_to_target_fps,
|
||||
)
|
||||
|
||||
FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000]
|
||||
SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120]
|
||||
|
||||
|
||||
def clip_to_aligned_frames(video):
|
||||
video = video[:121]
|
||||
if not len(video):
|
||||
raise ValueError("empty video")
|
||||
return video[: 1 + 8 * ((len(video) - 1) // 8)]
|
||||
|
||||
|
||||
def _reference_resample(video, fps):
|
||||
if abs(fps - 24) < 1.5 or fps < 24:
|
||||
return video, round(fps)
|
||||
step = fps / 24
|
||||
indices = [round(i * step) for i in range(int(len(video) / step))]
|
||||
return video[[i for i in indices if i < len(video)]], 24
|
||||
|
||||
|
||||
def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
|
||||
"""Run the streaming selector over frames ``0 .. total - 1`` (early stop included)."""
|
||||
selector = FrameSelector(fps)
|
||||
@@ -54,7 +65,7 @@ def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
|
||||
|
||||
def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None:
|
||||
ids = torch.arange(total).view(total, 1, 1, 1)
|
||||
resampled, effective_fps = resample_to_target_fps(ids, fps)
|
||||
resampled, effective_fps = _reference_resample(ids, fps)
|
||||
try:
|
||||
clipped = clip_to_aligned_frames(resampled)
|
||||
except ValueError:
|
||||
@@ -147,7 +158,7 @@ def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames):
|
||||
path = tmp_path / "clip.mp4"
|
||||
write_test_video(path, frames=frames, fps=fps)
|
||||
full, source_fps = _decode_everything(path)
|
||||
expected, expected_fps = resample_to_target_fps(full, source_fps)
|
||||
expected, expected_fps = _reference_resample(full, source_fps)
|
||||
expected = clip_to_aligned_frames(expected)
|
||||
|
||||
clip = decode_clip(str(path))
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import pytest
|
||||
import torch
|
||||
@@ -81,6 +82,35 @@ class ImageVAE:
|
||||
return SimpleNamespace(sample=lambda generator=None: self.latent)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_size", [(1920, 1080), (1080, 1920), (288, 192)])
|
||||
@pytest.mark.parametrize("tensor_input", [False, True], ids=["pil", "tensor"])
|
||||
def test_image_preprocessing_preserves_aspect_and_center_crops(
|
||||
source_size, tensor_input
|
||||
):
|
||||
src_w, src_h = source_size
|
||||
height, width = 64, 96
|
||||
pixels = np.full((src_h, src_w, 3), 128, dtype=np.uint8)
|
||||
pixels[..., 0] = np.linspace(0, 255, src_w, dtype=np.uint8)[None, :]
|
||||
pixels[..., 1] = np.linspace(0, 255, src_h, dtype=np.uint8)[:, None]
|
||||
image = PIL.Image.fromarray(pixels)
|
||||
|
||||
# independent PIL oracle for the reference resize + center-crop geometry
|
||||
scale = min(src_h / height, src_w / width)
|
||||
new_h, new_w = int(src_h / scale), int(src_w / scale)
|
||||
top, left = (new_h - height) // 2, (new_w - width) // 2
|
||||
expected = image.resize((new_w, new_h), PIL.Image.Resampling.BILINEAR).crop(
|
||||
(left, top, left + width, top + height)
|
||||
)
|
||||
expected = torch.from_numpy(np.array(expected)).permute(2, 0, 1).float() / 127.5 - 1
|
||||
source = (
|
||||
torch.from_numpy(pixels).permute(2, 0, 1).float() / 127.5 - 1
|
||||
if tensor_input
|
||||
else image
|
||||
)
|
||||
actual = Kandinsky6ImageEncodingStage._preprocess(source, height, width)
|
||||
torch.testing.assert_close(actual, expected.unsqueeze(0), atol=2 / 255, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 2])
|
||||
def test_image_request_broadcasts_one_reference_to_each_output(server_args, batch_size):
|
||||
request = VideoGenerationsRequest(
|
||||
|
||||
Reference in New Issue
Block a user