From 1351f32bb4c41427967c8082622af2cd9627ff65 Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 7 Oct 2026 08:48:16 +0800 Subject: [PATCH] [diffusion] refactor: simplify Kandinsky 6 models, SR sampling and tests (#42768) Co-authored-by: Mick Qian --- .../diffusion/Kandinsky/Kandinsky6-SR.mdx | 4 +- .../configs/models/dits/kandinsky6_sr.py | 178 ++--- .../configs/pipeline_configs/kandinsky6.py | 24 +- .../runtime/models/dits/kandinsky6.py | 631 ++++++----------- .../runtime/models/dits/kandinsky6_sr.py | 211 ++---- .../models/schedulers/kandinsky6_piflow.py | 163 +---- .../kandinsky6_sr_latent_upscaler.py | 71 +- .../runtime/models/vaes/kandinsky6_audio.py | 13 +- .../vaes/kandinsky6_audio_vae/bigvgan.py | 385 +++-------- .../vaes/kandinsky6_audio_vae/mmaudio_vae.py | 648 ++---------------- .../runtime/models/vaes/kandinsky6_sr_vae.py | 79 +-- .../pipelines/kandinsky6_sr_pipeline.py | 12 +- .../kandinsky6/denoising.py | 68 +- .../kandinsky6_sr/__init__.py | 8 +- .../kandinsky6_sr/decode_stage.py | 78 --- .../kandinsky6_sr/denoising_stage.py | 137 ---- .../kandinsky6_sr/encode_stage.py | 65 -- .../kandinsky6_sr/input_stage.py | 95 --- .../kandinsky6_sr/latent_prep_stage.py | 162 ----- .../kandinsky6_sr/latents.py | 104 +-- .../kandinsky6_sr/output_stage.py | 72 -- .../kandinsky6_sr/run_spec.py | 190 ++--- .../kandinsky6_sr/sampling.py | 216 +++--- .../kandinsky6_sr/stages.py | 463 +++++++++++++ .../kandinsky6_sr/tiled.py | 220 ++---- .../kandinsky6_sr/tiling.py | 47 +- .../kandinsky6_sr/video_io.py | 85 ++- .../kandinsky6_sr/video_utils.py | 136 ---- .../multimodal_gen/test/unit/conftest.py | 21 + .../unit/kandinsky6_sr_tiny_components.py | 71 +- .../test_kandinsky6_audio_vae_weight_norm.py | 17 +- .../test/unit/test_kandinsky6_bf16_dtype.py | 148 +--- ...st_kandinsky6_image_conditioning_resize.py | 127 ---- .../unit/test_kandinsky6_layerwise_offload.py | 18 +- .../unit/test_kandinsky6_pipeline_contract.py | 70 +- .../test/unit/test_kandinsky6_rope.py | 2 +- .../test/unit/test_kandinsky6_sr_config.py | 105 +-- .../test_kandinsky6_sr_decode_parallel.py | 6 +- .../unit/test_kandinsky6_sr_http_request.py | 285 ++------ .../test/unit/test_kandinsky6_sr_loading.py | 222 ++---- ..._kandinsky6_sr_official_release_configs.py | 105 ++- .../test_kandinsky6_sr_reference_parity.py | 200 ++---- .../test_kandinsky6_sr_sampler_selection.py | 253 ------- .../test/unit/test_kandinsky6_sr_stages.py | 115 ++-- .../test/unit/test_kandinsky6_sr_tiling.py | 23 +- .../test/unit/test_kandinsky6_sr_video_io.py | 29 +- .../test/unit/test_kandinsky6_stages.py | 30 + 47 files changed, 1792 insertions(+), 4620 deletions(-) delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/stages.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py delete mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py delete mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py diff --git a/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx b/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx index fc6fdd9f48..14ba1c7336 100644 --- a/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx +++ b/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx @@ -46,7 +46,9 @@ the source soundtrack rather than generating audio. Audio is converted to mono Use `kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers` for x2, x4 or x2.25 upscaling of up to 121 frames. Its checkpoint-selected PiFlow scheduler uses two DiT calls per tile; changing `num_inference_steps` does not increase -quality or the number of evaluations. The native path uses dense attention, +quality or the number of evaluations. The scheduler and transformer must come +from the same Diffusers checkpoint; sampler settings are read from `scheduler/`. +The native path uses dense attention, not the checkpoint's requested NABLA sparse attention, so this is not a claim of numerical parity with the sparse reference. diff --git a/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py index 81f9f092b4..814ec78318 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py +++ b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py @@ -4,7 +4,7 @@ Official out_visual_dim is the total DX head width; n_grid lives in the scheduler. Unsupported architecture options and unknown fields are rejected.""" -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from typing import Any from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig @@ -16,13 +16,6 @@ ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_param # Bookkeeping keys the loader leaves in the component config. _TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"}) _SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal") -_PIFLOW_FIELDS = ( - "piflow_nfe", - "piflow_num_policy_substeps", - "piflow_final_step_size_scale", - "piflow_shift", - "piflow_eps", -) _LQ_NOISE_TYPES = ("ddpm", "linear") @@ -40,7 +33,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig): } ) - # --- DiffusionTransformer3D constructor keys (trained architecture) --- + # trained architecture, before inference overrides in_visual_dim: int = 4 in_text_dim: int = 3584 in_text_dim2: int = 768 @@ -61,30 +54,13 @@ class Kandinsky6SRArchConfig(DiTArchConfig): zero_lq_in_main_path: bool = False use_adapter: bool = False adapter_gamma: float = 0.0 - # The reference default is True; the SR release is text-free and its config says - # so explicitly. use_text: bool = False use_lq_noise_cond: bool = False - # --- optional legacy / test-only DX head / pi-Flow fields --- - # An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the - # pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is - # ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim). - n_grid: int = 1 - piflow_nfe: int | None = None - piflow_num_policy_substeps: int | None = None - piflow_final_step_size_scale: float | None = None - piflow_shift: float | None = None - piflow_eps: float | None = None - - # Post-load attribute overrides (whitelist: instruct_type, visual_cond, - # attention_params); the architecture above stays as trained. The official - # config writes ``null`` when there are none. + # inference-only overrides must not change the trained layer dimensions attribute_overrides: dict[str, Any] | None = field(default_factory=dict) - # --- SR sampling parameters of the training config --- - # As stored in ``transformer/config.json`` (``sr_params``); expanded into the - # flat ``sr_*`` fields below in ``__post_init__``. + # checkpoint sr_params expands into the flat sr_* fields sr_params: dict[str, Any] | None = None sr_visual_size: list[int] = field(default_factory=lambda: [512]) sr_scale_factor: dict[str, list[float]] | None = None @@ -104,77 +80,18 @@ class Kandinsky6SRArchConfig(DiTArchConfig): if self.attribute_overrides is None: self.attribute_overrides = {} if self.sr_params is not None: - self._expand_sr_params(self.sr_params) - self._reject_unknown_keys() - self._reject_unsupported_flags() - self._normalize_and_validate_shapes() - self._validate_overrides() - self._validate_sampler_fields() - self.hidden_size = self.model_dim - self.num_attention_heads = self.model_dim // sum(self.axes_dims) - self.in_channels = self.in_visual_dim - self.out_channels = self.out_visual_dim - self.num_channels_latents = self.in_visual_dim - - @property - def is_piflow(self) -> bool: - return self.piflow_nfe is not None - - @property - def head_width(self) -> int: - """Channels the DiT head emits per latent position (all grids of a DX head).""" - return self.out_visual_dim * self.n_grid - - @property - def base_out_visual_dim(self) -> int: - """Channels per latent grid; legacy n_grid configs store the base output width.""" - return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim - - @property - def trained_wide_input(self) -> bool: - """Whether the input layer was trained on ``[x | cond | mask]`` channels.""" - return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES) - - def effective_override(self, name: str) -> Any: - """Post-load value of ``name``: the override if present, else as trained.""" - if name in self.attribute_overrides: - return self.attribute_overrides[name] - return getattr(self, name) - - def requested_sparse_attention(self) -> str | None: - """Sparse attention type asked for by ``attention_params``, if any.""" - params = self.effective_override("attention_params") - if not isinstance(params, dict): - return None - for value in params.values(): - attention_type = value.get("type") if isinstance(value, dict) else None - if attention_type in _SPARSE_ATTENTION_TYPES: - return attention_type - return None - - _SR_PARAM_FIELDS = { - "visual_size": "sr_visual_size", - "scale_factor": "sr_scale_factor", - "scheduler_scale": "sr_scheduler_scale", - "lq_noise_scale": "sr_lq_noise_scale", - "lq_noise_type": "sr_lq_noise_type", - "lq_channel_noise_scale": "sr_lq_channel_noise_scale", - "cap_noise_timestep": "sr_cap_noise_timestep", - "fps": "sr_fps", - } - - def _expand_sr_params(self, sr_params: dict[str, Any]) -> None: - """Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields.""" - unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS)) - if unknown: - raise ValueError( - f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted " - f"keys: {sorted(self._SR_PARAM_FIELDS)}" - ) - for key, value in sr_params.items(): - setattr(self, self._SR_PARAM_FIELDS[key], value) - - def _reject_unknown_keys(self) -> None: + known = { + item.name.removeprefix("sr_") + for item in fields(self) + if item.name.startswith("sr_") and item.name != "sr_params" + } + unknown = sorted(set(self.sr_params) - known) + if unknown: + raise ValueError( + f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted keys: {sorted(known)}" + ) + for key, value in self.sr_params.items(): + setattr(self, f"sr_{key}", value) unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS) if unknown: raise ValueError( @@ -182,23 +99,15 @@ class Kandinsky6SRArchConfig(DiTArchConfig): f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config." ) - def _reject_unsupported_flags(self) -> None: - if self.use_text: - raise NotImplementedError( - "Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not " - "supported; only the text-free SR DiT (use_text=False) is implemented." - ) - if self.use_adapter: - raise NotImplementedError( - "Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported." - ) - if self.use_lq_modulation: - raise NotImplementedError( - "Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is " - "not supported." - ) - - def _normalize_and_validate_shapes(self) -> None: + for name, enabled in ( + ("use_text", self.use_text), + ("use_adapter", self.use_adapter), + ("use_lq_modulation", self.use_lq_modulation), + ): + if enabled: + raise NotImplementedError( + f"Kandinsky6SR: {name}=True is not supported." + ) self.patch_size = tuple(self.patch_size) self.axes_dims = tuple(self.axes_dims) if len(self.patch_size) != 3 or self.patch_size[0] != 1: @@ -218,10 +127,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig): ) if self.model_dim % 2 != 0: raise ValueError(f"model_dim must be even, got {self.model_dim}") - if self.n_grid < 1: - raise ValueError(f"n_grid must be >= 1, got {self.n_grid}") - def _validate_overrides(self) -> None: unknown = sorted( set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST) ) @@ -239,19 +145,37 @@ class Kandinsky6SRArchConfig(DiTArchConfig): visual_cond = self.effective_override("visual_cond") if not isinstance(visual_cond, bool): raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}") - - def _validate_sampler_fields(self) -> None: - present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None] - if present and len(present) != len(_PIFLOW_FIELDS): - missing = sorted(set(_PIFLOW_FIELDS) - set(present)) - raise ValueError( - f"pi-Flow config is incomplete: {missing} missing next to {present}" - ) if self.sr_lq_noise_type not in _LQ_NOISE_TYPES: raise ValueError( f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, " f"got {self.sr_lq_noise_type!r}" ) + self.hidden_size = self.model_dim + self.num_attention_heads = self.model_dim // head_dim + self.in_channels = self.in_visual_dim + self.out_channels = self.out_visual_dim + self.num_channels_latents = self.in_visual_dim + + @property + def trained_wide_input(self) -> bool: + """Whether the input layer was trained on ``[x | cond | mask]`` channels.""" + return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES) + + def effective_override(self, name: str) -> Any: + """Post-load value of ``name``: the override if present, else as trained.""" + return self.attribute_overrides.get(name, self.__dict__[name]) + + def requested_sparse_attention(self) -> str | None: + """Sparse attention type asked for by ``attention_params``, if any.""" + params = self.effective_override("attention_params") + if isinstance(params, dict): + for value in params.values(): + if ( + isinstance(value, dict) + and value.get("type") in _SPARSE_ATTENTION_TYPES + ): + return value["type"] + return None @dataclass diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py index 80a27e8d4c..6d4534d5a8 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py @@ -157,21 +157,15 @@ class Kandinsky6TI2VAPipelineConfig(PipelineConfig): audio_downsample_factor: int = 1024 def __post_init__(self) -> None: - if len(self.text_encoder_configs) != 2: - raise ValueError( - "Kandinsky6 pipeline requires exactly 2 text encoders " - f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}." - ) - if len(self.text_encoder_precisions) != 2: - raise ValueError( - "Kandinsky6 pipeline requires exactly 2 text encoder " - f"precisions, but got {len(self.text_encoder_precisions)}." - ) - if len(self.text_encoder_extra_args) != 2: - raise ValueError( - "Kandinsky6 pipeline requires exactly 2 text encoder extra " - f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}." - ) + for name, values in ( + ("text encoders (Reason1 and CLIP)", self.text_encoder_configs), + ("text encoder precisions", self.text_encoder_precisions), + ("text encoder extra tokenizer arg dicts", self.text_encoder_extra_args), + ): + if len(values) != 2: + raise ValueError( + f"Kandinsky6 pipeline requires exactly 2 {name}, but got {len(values)}." + ) # keep the video VAE encoder available for optional image conditioning self.vae_config.load_encoder = True diff --git a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py index 21a9713348..dbb39879ed 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py @@ -10,11 +10,14 @@ NABLA attention is not supported. import math from dataclasses import dataclass +from functools import partial from typing import Any import torch import torch.nn as nn import torch.nn.functional as F +from einops import rearrange +from torch.utils.checkpoint import checkpoint from sglang.kernels.ops.diffusion import apply_matrix_rope, residual_gate_fp32 from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( @@ -69,6 +72,28 @@ def _is_kandinsky6_transformer_block(name: str, module: object) -> bool: return is_module_list_entry_in(name, _KANDINSKY6_BLOCK_CONTAINERS) +def _validate_parallelism(num_heads: int, **tp_dimensions: int) -> None: + """TP splits heads/FFNs; Ulysses splits video heads remaining after TP.""" + tp_size = get_tp_world_size() + ulysses_size, _ = get_ulysses_ctx() + ring_size, _ = get_ring_ctx() + for name, size in (("TP", tp_size), ("Ulysses", ulysses_size), ("ring", ring_size)): + if size <= 0: + raise ValueError(f"Kandinsky6 {name} size must be positive.") + for name, value in dict(num_attention_heads=num_heads, **tp_dimensions).items(): + if value % tp_size: + raise ValueError( + f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}." + ) + # ring rotates complete K/V shards, so it imposes no head divisibility constraint + local_heads = num_heads // tp_size + if local_heads % ulysses_size: + raise ValueError( + f"Kandinsky6 TP-local video attention heads {local_heads} must be " + f"divisible by Ulysses size {ulysses_size} (total heads={num_heads}, TP={tp_size})." + ) + + def _build_rotary_freqs(dim: int, max_period: float) -> torch.Tensor: return torch.exp( -math.log(max_period) @@ -146,20 +171,13 @@ class Kandinsky6VisualEmbeddings(nn.Module): ) def forward(self, x: torch.Tensor) -> torch.Tensor: - batch_size, duration, height, width, dim = x.shape - x = ( - x.view( - batch_size, - duration // self.patch_size[0], - self.patch_size[0], - height // self.patch_size[1], - self.patch_size[1], - width // self.patch_size[2], - self.patch_size[2], - dim, - ) - .permute(0, 1, 3, 5, 2, 4, 6, 7) - .flatten(4, 7) + pt, ph, pw = self.patch_size + x = rearrange( + x, + "b (t pt) (h ph) (w pw) c -> b t h w (pt ph pw c)", + pt=pt, + ph=ph, + pw=pw, ) x, _ = self.in_layer(x) return x @@ -321,30 +339,22 @@ class Kandinsky6Attention(nn.Module): tp_size = get_tp_world_size() self.local_num_heads = divide(self.num_heads, tp_size) - self.to_query = ColumnParallelLinear( - num_channels, - num_channels, - bias=True, - gather_output=False, - quant_config=quant_config, - prefix=add_prefix("to_query", prefix), - ) - self.to_key = ColumnParallelLinear( - kv_dim, - num_channels, - bias=True, - gather_output=False, - quant_config=quant_config, - prefix=add_prefix("to_key", prefix), - ) - self.to_value = ColumnParallelLinear( - kv_dim, - num_channels, - bias=True, - gather_output=False, - quant_config=quant_config, - prefix=add_prefix("to_value", prefix), - ) + for name, width in ( + ("to_query", num_channels), + ("to_key", kv_dim), + ("to_value", kv_dim), + ): + self.add_module( + name, + ColumnParallelLinear( + width, + num_channels, + bias=True, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix(name, prefix), + ), + ) self.query_norm = Kandinsky6QKNorm(head_dim) self.key_norm = Kandinsky6QKNorm(head_dim) self.out_layer = RowParallelLinear( @@ -414,48 +424,20 @@ class Kandinsky6Attention(nn.Module): key = gather_seq(key, context_seq_len) value = gather_seq(value, context_seq_len) - try: - hidden_states = self.attention( - query, - key, - value, - attn_mask_meta=attn_mask_meta, - skip_sequence_parallel_override=skip_sequence_parallel, - ) - except AssertionError as exc: - # standalone single-rank parity uses SDPA without a pipeline forward context - if "Forward context is not set" not in str(exc): - raise - - query_shape = query.shape[:-2] - key_shape = key.shape[:-2] - query = query.reshape( - query_shape[0], -1, self.local_num_heads, query.shape[-1] - ).transpose(1, 2) - key = key.reshape( - key_shape[0], -1, self.local_num_heads, key.shape[-1] - ).transpose(1, 2) - value = value.reshape( - key_shape[0], -1, self.local_num_heads, value.shape[-1] - ).transpose(1, 2) - hidden_states = F.scaled_dot_product_attention( - query, - key, - value, - attn_mask=None, - is_causal=False, - ) - hidden_states = hidden_states.transpose(1, 2).reshape( - *query_shape, self.local_num_heads, -1 - ) - + hidden_states = self.attention( + query, + key, + value, + attn_mask_meta=attn_mask_meta, + skip_sequence_parallel_override=skip_sequence_parallel, + ) hidden_states = hidden_states.flatten(-2, -1) hidden_states, _ = self.out_layer(hidden_states) return hidden_states -class _Kandinsky6MLP(nn.Module): - """Bias-free TP MLP with checkpoint-mapped fc_in/fc_out names. +class Kandinsky6FeedForward(nn.Module): + """Bias-free TP MLP with checkpoint-mapped mlp.fc_in/fc_out names. The shared MLP currently hardcodes bias=True, so it cannot be used here.""" @@ -467,7 +449,9 @@ class _Kandinsky6MLP(nn.Module): quant_config: QuantizationConfig | None = None, ): super().__init__() - self.fc_in = ColumnParallelLinear( + prefix = add_prefix("mlp", prefix) + self.mlp = nn.Module() + self.mlp.fc_in = ColumnParallelLinear( dim, ff_dim, bias=False, @@ -475,8 +459,8 @@ class _Kandinsky6MLP(nn.Module): quant_config=quant_config, prefix=add_prefix("fc_in", prefix), ) - self.act = get_act_fn("gelu") - self.fc_out = RowParallelLinear( + self.mlp.act = get_act_fn("gelu") + self.mlp.fc_out = RowParallelLinear( ff_dim, dim, bias=False, @@ -486,29 +470,12 @@ class _Kandinsky6MLP(nn.Module): ) def forward(self, x: torch.Tensor) -> torch.Tensor: - x, _ = self.fc_in(x) - x = self.act(x) - x, _ = self.fc_out(x) + x, _ = self.mlp.fc_in(x) + x = self.mlp.act(x) + x, _ = self.mlp.fc_out(x) return x -class Kandinsky6FeedForward(nn.Module): - def __init__( - self, - dim: int, - ff_dim: int, - prefix: str = "", - quant_config: QuantizationConfig | None = None, - ): - super().__init__() - self.mlp = _Kandinsky6MLP( - dim, ff_dim, prefix=add_prefix("mlp", prefix), quant_config=quant_config - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.mlp(x) - - def _norm_scale_shift( norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor ) -> torch.Tensor: @@ -535,7 +502,11 @@ class Kandinsky6OutLayer(nn.Module): ) def forward( - self, visual_embed: torch.Tensor, time_embed: torch.Tensor + self, + visual_embed: torch.Tensor, + time_embed: torch.Tensor, + *, + compute_dtype: torch.dtype | None = None, ) -> torch.Tensor: shift, scale = torch.chunk( self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 @@ -543,28 +514,18 @@ class Kandinsky6OutLayer(nn.Module): visual_embed = ( self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0) + shift.float()[:, None, None] - ).type_as(visual_embed) + ).to(dtype=compute_dtype or visual_embed.dtype) x, _ = self.out_layer(visual_embed) - batch_size, duration, height, width, _ = x.shape - x = ( - x.view( - batch_size, - duration, - height, - width, - -1, - self.patch_size[0], - self.patch_size[1], - self.patch_size[2], - ) - .permute(0, 1, 5, 2, 6, 3, 7, 4) - .flatten(1, 2) - .flatten(2, 3) - .flatten(3, 4) + pt, ph, pw = self.patch_size + return rearrange( + x, + "b t h w (c pt ph pw) -> b (t pt) (h ph) (w pw) c", + pt=pt, + ph=ph, + pw=pw, ) - return x class Kandinsky6OutLayerAudio(nn.Module): @@ -591,8 +552,12 @@ class Kandinsky6OutLayerAudio(nn.Module): return out -class Kandinsky6TransformerEncoderBlock(nn.Module): - """Text-only self-attention + feed-forward block (video/audio text towers).""" +class Kandinsky6TransformerBlock(nn.Module): + """Shared parameter layout for text, video and SR transformer blocks.""" + + modulation_name = "visual_modulation" + with_cross_attention = False + skip_sequence_parallel = False def __init__( self, @@ -605,24 +570,38 @@ class Kandinsky6TransformerEncoderBlock(nn.Module): quant_config: QuantizationConfig | None = None, ): super().__init__() - self.text_modulation = Kandinsky6Modulation(time_dim, model_dim, 6) - - self.self_attention_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + self.add_module( + self.modulation_name, + Kandinsky6Modulation( + time_dim, model_dim, 9 if self.with_cross_attention else 6 + ), ) - self.self_attention = Kandinsky6Attention( - model_dim, - head_dim, - supported_attention_backends=supported_attention_backends, - prefix=add_prefix("self_attention", prefix), - quant_config=quant_config, - # text is replicated across SP ranks, so skip Ulysses even for self-attention - skip_sequence_parallel=True, - ) - - self.feed_forward_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + attention_names = ( + ("self_attention", "cross_attention") + if self.with_cross_attention + else ("self_attention",) ) + for name in (*attention_names, "feed_forward"): + self.add_module( + f"{name}_norm", + LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ), + ) + for name in attention_names: + self.add_module( + name, + Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends, + prefix=add_prefix(name, prefix), + quant_config=quant_config, + is_cross_attention=name == "cross_attention", + skip_sequence_parallel=self.skip_sequence_parallel + or name == "cross_attention", + ), + ) self.feed_forward = Kandinsky6FeedForward( model_dim, ff_dim, @@ -630,6 +609,13 @@ class Kandinsky6TransformerEncoderBlock(nn.Module): quant_config=quant_config, ) + +class Kandinsky6TransformerEncoderBlock(Kandinsky6TransformerBlock): + """Replicated text self-attention and FFN.""" + + modulation_name = "text_modulation" + skip_sequence_parallel = True + def forward( self, x: torch.Tensor, time_embed: torch.Tensor, rope: torch.Tensor ) -> torch.Tensor: @@ -649,59 +635,10 @@ class Kandinsky6TransformerEncoderBlock(nn.Module): return x -class Kandinsky6TransformerDecoderBlock(nn.Module): - """Self-attention, text cross-attention and FFN. +class Kandinsky6TransformerDecoderBlock(Kandinsky6TransformerBlock): + """Video self-attention, text cross-attention and FFN.""" - The joint decoder invokes these sublayers separately to insert cross-modal attention.""" - - def __init__( - self, - model_dim: int, - time_dim: int, - ff_dim: int, - head_dim: int, - supported_attention_backends: set[AttentionBackendEnum] | None = None, - prefix: str = "", - quant_config: QuantizationConfig | None = None, - ): - super().__init__() - self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 9) - - self.self_attention_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 - ) - self.self_attention = Kandinsky6Attention( - model_dim, - head_dim, - supported_attention_backends=supported_attention_backends, - prefix=add_prefix("self_attention", prefix), - quant_config=quant_config, - # Video self-attention over the (long) visual token sequence -- - # the one role this port's sequence parallelism actually shards. - ) - - self.cross_attention_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 - ) - self.cross_attention = Kandinsky6Attention( - model_dim, - head_dim, - supported_attention_backends=supported_attention_backends, - prefix=add_prefix("cross_attention", prefix), - quant_config=quant_config, - # video queries attend to replicated text K/V - is_cross_attention=True, - ) - - self.feed_forward_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 - ) - self.feed_forward = Kandinsky6FeedForward( - model_dim, - ff_dim, - prefix=add_prefix("feed_forward", prefix), - quant_config=quant_config, - ) + with_cross_attention = True def forward( self, @@ -749,12 +686,6 @@ class Kandinsky6TransformerDecoderBlock(nn.Module): return visual_embed -def _apply_gate_sum( - x: torch.Tensor, out: torch.Tensor, gate: torch.Tensor -) -> torch.Tensor: - return residual_gate_fp32(x, out, gate) - - def _apply_scale_shift( norm: nn.LayerNorm, x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor ) -> torch.Tensor: @@ -856,7 +787,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): self.videoT.visual_modulation(t_v).unsqueeze(dim=1), 3, dim=-1 ) shift, scale, gate = torch.chunk(sa_p, 3, dim=-1) - vis = _apply_gate_sum( + vis = residual_gate_fp32( vis, self.videoT.self_attention( _norm_scale_shift(self.videoT.self_attention_norm, vis, shift, scale), @@ -877,7 +808,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): self.audioT.visual_modulation(t_a).unsqueeze(dim=1), 3, dim=-1 ) shift, scale, gate = torch.chunk(sa_p, 3, dim=-1) - aud = _apply_gate_sum( + aud = residual_gate_fp32( aud, self.audioT.self_attention( _norm_scale_shift(self.audioT.self_attention_norm, aud, shift, scale), @@ -893,7 +824,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): aud_out_t = self.audioT.cross_attention( aud_pre_ca, encoder_hidden_states=text_a ) - aud = _apply_gate_sum(aud, aud_out_t, gate_a) + aud = residual_gate_fp32(aud, aud_out_t, gate_a) t_va_mod = t_a if not self.fix_modulation else t_v t_av_mod = t_v if not self.fix_modulation else t_a @@ -910,7 +841,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): va_shift, va_scale, va_gate = torch.chunk(va_params, 3, dim=-1) av_shift, av_scale, av_gate = torch.chunk(av_params, 3, dim=-1) - vis = _apply_gate_sum(vis, vis_out_t, gate_v) + vis = residual_gate_fp32(vis, vis_out_t, gate_v) vis_for_va = _apply_scale_shift(self.va_normalization, vis, va_scale, va_shift) aud_for_av = _apply_scale_shift(self.av_normalization, aud, av_scale, av_shift) rq_v = vis_rope if self.ca_rope else None @@ -928,19 +859,19 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): rotary_emb_kv=rq_v, context_seq_len=video_seq_len, ) - vis = _apply_gate_sum( + vis = residual_gate_fp32( vis, vis_from_aud, (va_gate if not self.cross_gates else av_gate) * va_gate_scale, ) - aud = _apply_gate_sum( + aud = residual_gate_fp32( aud, aud_from_vis, (av_gate if not self.cross_gates else va_gate) * av_gate_scale, ) shift, scale, gate = torch.chunk(ff_p, 3, dim=-1) - vis = _apply_gate_sum( + vis = residual_gate_fp32( vis, self.videoT.feed_forward( _norm_scale_shift(self.videoT.feed_forward_norm, vis, shift, scale) @@ -948,7 +879,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module): gate, ) shift, scale, gate = torch.chunk(ff_p_a, 3, dim=-1) - aud = _apply_gate_sum( + aud = residual_gate_fp32( aud, self.audioT.feed_forward( _norm_scale_shift(self.audioT.feed_forward_norm, aud, shift, scale) @@ -977,42 +908,6 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): AttentionBackendEnum.TORCH_SDPA, } - @staticmethod - def _validate_tp_config( - *, arch: Kandinsky6ArchConfig, tp_size: int, num_heads: int, num_heads_a: int - ) -> None: - if tp_size <= 0: - raise ValueError("Kandinsky6 TP size must be positive.") - for name, value in ( - ("num_attention_heads (video)", num_heads), - ("num_attention_heads_a (audio)", num_heads_a), - ("ff_dim", arch.ff_dim), - ("ff_dim_a", arch.ff_dim_a), - ): - if value % tp_size: - raise ValueError( - f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}." - ) - - @staticmethod - def _validate_sequence_parallel_config( - *, tp_size: int, num_heads: int, ulysses_size: int, ring_size: int - ) -> None: - if ulysses_size <= 0: - raise ValueError("Kandinsky6 Ulysses size must be positive.") - if ring_size <= 0: - raise ValueError("Kandinsky6 ring size must be positive.") - if ulysses_size == 1 and ring_size == 1: - return - # only Ulysses splits heads; ring rotates complete K/V shards - local_heads = num_heads // tp_size - if local_heads % ulysses_size: - raise ValueError( - f"Kandinsky6 TP-local video attention heads {local_heads} must be " - f"divisible by Ulysses size {ulysses_size} (total video heads=" - f"{num_heads}, TP={tp_size})." - ) - def __init__( self, config: Kandinsky6VideoAudioConfig, @@ -1034,20 +929,11 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): head_dim = sum(arch.axes_dims) head_dim_a = sum(arch.axes_dims_a) - tp_size = get_tp_world_size() - ulysses_size, _ = get_ulysses_ctx() - ring_size, _ = get_ring_ctx() - self._validate_tp_config( - arch=arch, - tp_size=tp_size, - num_heads=arch.model_dim // head_dim, - num_heads_a=arch.model_dim_a // head_dim_a, - ) - self._validate_sequence_parallel_config( - tp_size=tp_size, - num_heads=arch.model_dim // head_dim, - ulysses_size=ulysses_size, - ring_size=ring_size, + _validate_parallelism( + arch.model_dim // head_dim, + num_attention_heads_a=arch.model_dim_a // head_dim_a, + ff_dim=arch.ff_dim, + ff_dim_a=arch.ff_dim_a, ) self.in_visual_dim = arch.in_visual_dim self.in_audio_dim = arch.in_audio_dim @@ -1079,46 +965,10 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): ) if not self.is_multimodal: - self.time_embeddings = Kandinsky6TimeEmbeddings( - arch.model_dim, arch.time_dim - ) - self.text_embeddings = Kandinsky6TextEmbeddings( - arch.in_text_dim, arch.model_dim - ) - self.pooled_text_embeddings = Kandinsky6TextEmbeddings( - arch.in_text_dim2, arch.time_dim - ) - self.text_rope_embeddings = Kandinsky6RoPE1D(head_dim) - self.text_transformer_blocks = nn.ModuleList( - [ - Kandinsky6TransformerEncoderBlock( - arch.model_dim, - arch.time_dim, - arch.ff_dim, - head_dim, - self._supported_attention_backends, - prefix=add_prefix(f"text_transformer_blocks.{i}", self.prefix), - quant_config=quant_config, - ) - for i in range(arch.num_text_blocks) - ] - ) - self.visual_transformer_blocks = nn.ModuleList( - [ - Kandinsky6TransformerDecoderBlock( - arch.model_dim, - arch.time_dim, - arch.ff_dim, - head_dim, - self._supported_attention_backends, - prefix=add_prefix( - f"visual_transformer_blocks.{i}", self.prefix - ), - quant_config=quant_config, - ) - for i in range(arch.num_visual_blocks) - ] + self._build_text_tower( + "", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim ) + block_cls = Kandinsky6TransformerDecoderBlock else: self.audio_embeddings = Kandinsky6TextEmbeddings( arch.in_audio_dim, arch.model_dim_a @@ -1132,81 +982,35 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): arch.out_audio_dim or arch.in_audio_dim, ) - for ( - tower_prefix, - tower_model_dim, - tower_time_dim, - tower_head_dim, - tower_ff_dim, - ) in ( - ("video", arch.model_dim, arch.time_dim, head_dim, arch.ff_dim), - ("audio", arch.model_dim_a, arch.time_dim_a, head_dim_a, arch.ff_dim_a), - ): - setattr( - self, - f"{tower_prefix}_time_embeddings", - Kandinsky6TimeEmbeddings(tower_model_dim, tower_time_dim), - ) - setattr( - self, - f"{tower_prefix}_text_embeddings", - Kandinsky6TextEmbeddings(arch.in_text_dim, tower_model_dim), - ) - setattr( - self, - f"{tower_prefix}_pooled_text_embeddings", - Kandinsky6TextEmbeddings(arch.in_text_dim2, tower_time_dim), - ) - setattr( - self, - f"{tower_prefix}_text_rope_embeddings", - Kandinsky6RoPE1D(tower_head_dim), - ) - setattr( - self, - f"{tower_prefix}_text_transformer_blocks", - nn.ModuleList( - [ - Kandinsky6TransformerEncoderBlock( - tower_model_dim, - tower_time_dim, - tower_ff_dim, - tower_head_dim, - self._supported_attention_backends, - prefix=add_prefix( - f"{tower_prefix}_text_transformer_blocks.{i}", - self.prefix, - ), - quant_config=quant_config, - ) - for i in range(arch.num_text_blocks) - ] - ), - ) - - self.visual_transformer_blocks = nn.ModuleList( - [ - Kandinsky6FusedTransformerDecoderBlock( - arch.model_dim, - arch.time_dim, - arch.ff_dim, - head_dim, - arch.model_dim_a, - arch.time_dim_a, - arch.ff_dim_a, - head_dim_a, - self._supported_attention_backends, - prefix=add_prefix( - f"visual_transformer_blocks.{i}", self.prefix - ), - ca_rope=arch.ca_rope, - cross_gates=arch.cross_gates, - fix_modulation=arch.fix_modulation, - quant_config=quant_config, - ) - for i in range(arch.num_visual_blocks) - ] + self._build_text_tower( + "video_", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim ) + self._build_text_tower( + "audio_", arch.model_dim_a, arch.time_dim_a, arch.ff_dim_a, head_dim_a + ) + + block_cls = partial( + Kandinsky6FusedTransformerDecoderBlock, + model_dim_a=arch.model_dim_a, + time_dim_a=arch.time_dim_a, + ff_dim_a=arch.ff_dim_a, + head_dim_a=head_dim_a, + ca_rope=arch.ca_rope, + cross_gates=arch.cross_gates, + fix_modulation=arch.fix_modulation, + ) + self.visual_transformer_blocks = nn.ModuleList( + block_cls( + arch.model_dim, + arch.time_dim, + arch.ff_dim, + head_dim, + supported_attention_backends=self._supported_attention_backends, + prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix), + quant_config=quant_config, + ) + for i in range(arch.num_visual_blocks) + ) self.gradient_checkpointing = False self.hidden_size = arch.hidden_size @@ -1219,6 +1023,38 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): ) + ["visual_transformer_blocks"] self.__post_init__() + def _build_text_tower(self, prefix, model_dim, time_dim, ff_dim, head_dim): + arch = self.config + self.add_module( + f"{prefix}time_embeddings", Kandinsky6TimeEmbeddings(model_dim, time_dim) + ) + self.add_module( + f"{prefix}text_embeddings", + Kandinsky6TextEmbeddings(arch.in_text_dim, model_dim), + ) + self.add_module( + f"{prefix}pooled_text_embeddings", + Kandinsky6TextEmbeddings(arch.in_text_dim2, time_dim), + ) + self.add_module(f"{prefix}text_rope_embeddings", Kandinsky6RoPE1D(head_dim)) + self.add_module( + f"{prefix}text_transformer_blocks", + nn.ModuleList( + Kandinsky6TransformerEncoderBlock( + model_dim, + time_dim, + ff_dim, + head_dim, + self._supported_attention_backends, + prefix=add_prefix( + f"{prefix}text_transformer_blocks.{i}", self.prefix + ), + quant_config=self.quant_config, + ) + for i in range(arch.num_text_blocks) + ), + ) + def _encode_text( self, prefix: str | None, @@ -1237,11 +1073,8 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): text_rope = rope_embeddings(text_rope_pos).unsqueeze(dim=0) for block in blocks: if torch.is_grad_enabled() and self.gradient_checkpointing: - te = torch.utils.checkpoint.checkpoint( - block, te, tm, text_rope, use_reentrant=False - ) - else: - te = block(te, tm, text_rope) + block = partial(checkpoint, block, use_reentrant=False) + te = block(te, tm, text_rope) return te, tm def forward( @@ -1332,51 +1165,27 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): audio_rope = self.audio_rope_embeddings(audio_rope_pos).unsqueeze(dim=0) for block in self.visual_transformer_blocks: if torch.is_grad_enabled() and self.gradient_checkpointing: - visual_embed, audio_embed = torch.utils.checkpoint.checkpoint( - block, - visual_embed, - audio_embed, - video_te, - audio_te, - (video_tm, audio_tm), - visual_rope, - audio_rope, - va_gate_scale, - av_gate_scale, - video_seq_len=shard.orig_len, - video_attn_meta=attn_meta, - use_reentrant=False, - ) - else: - visual_embed, audio_embed = block( - visual_embed, - audio_embed, - video_te, - audio_te, - (video_tm, audio_tm), - visual_rope, - audio_rope, - va_gate_scale, - av_gate_scale, - video_seq_len=shard.orig_len, - video_attn_meta=attn_meta, - ) + block = partial(checkpoint, block, use_reentrant=False) + visual_embed, audio_embed = block( + visual_embed, + audio_embed, + video_te, + audio_te, + (video_tm, audio_tm), + visual_rope, + audio_rope, + va_gate_scale, + av_gate_scale, + video_seq_len=shard.orig_len, + video_attn_meta=attn_meta, + ) else: for block in self.visual_transformer_blocks: if torch.is_grad_enabled() and self.gradient_checkpointing: - visual_embed = torch.utils.checkpoint.checkpoint( - block, - visual_embed, - video_te, - video_tm, - visual_rope, - attn_mask_meta=attn_meta, - use_reentrant=False, - ) - else: - visual_embed = block( - visual_embed, video_te, video_tm, visual_rope, attn_meta - ) + block = partial(checkpoint, block, use_reentrant=False) + visual_embed = block( + visual_embed, video_te, video_tm, visual_rope, attn_meta + ) visual_embed = gather_seq(visual_embed, shard.orig_len).reshape( *visual_shape, -1 diff --git a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py index d13cd7fce5..ab84a67f6b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py @@ -12,16 +12,11 @@ import torch import torch.nn as nn from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( - ATTRIBUTE_OVERRIDE_WHITELIST, + WIDE_INPUT_INSTRUCT_TYPES, Kandinsky6SRArchConfig, Kandinsky6SRDitConfig, ) from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in -from sglang.multimodal_gen.runtime.distributed import get_tp_world_size -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - get_ring_ctx, - get_ulysses_ctx, -) from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import ( gather_seq, shard_like, @@ -38,14 +33,13 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im ) from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( - Kandinsky6Attention, - Kandinsky6FeedForward, - Kandinsky6Modulation, Kandinsky6OutLayer, Kandinsky6RoPE3D, Kandinsky6TimeEmbeddings, + Kandinsky6TransformerBlock, Kandinsky6VisualEmbeddings, _build_rotary_freqs, + _validate_parallelism, ) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -80,40 +74,8 @@ class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D): return self -class Kandinsky6SRDecoderBlock(nn.Module): - """Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears.""" - - def __init__( - self, - model_dim: int, - time_dim: int, - ff_dim: int, - head_dim: int, - supported_attention_backends: set[AttentionBackendEnum] | None = None, - prefix: str = "", - quant_config: QuantizationConfig | None = None, - ): - super().__init__() - self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6) - self.self_attention_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 - ) - self.self_attention = Kandinsky6Attention( - model_dim, - head_dim, - supported_attention_backends=supported_attention_backends, - prefix=add_prefix("self_attention", prefix), - quant_config=quant_config, - ) - self.feed_forward_norm = LayerNormScaleShift( - model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 - ) - self.feed_forward = Kandinsky6FeedForward( - model_dim, - ff_dim, - prefix=add_prefix("feed_forward", prefix), - quant_config=quant_config, - ) +class Kandinsky6SRDecoderBlock(Kandinsky6TransformerBlock): + """Modulated self-attention and FFN with fp32 residuals.""" def forward( self, @@ -139,32 +101,6 @@ class Kandinsky6SRDecoderBlock(nn.Module): return visual_embed + gate.float() * out.float() -class Kandinsky6SROutLayer(Kandinsky6OutLayer): - """K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype).""" - - def forward( - self, visual_embed: torch.Tensor, time_embed: torch.Tensor - ) -> torch.Tensor: - shift, scale = torch.chunk( - self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 - ) - x = ( - self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0) - + shift.float()[:, None, None] - ) - x, _ = self.out_layer(x.to(self.out_layer.weight.dtype)) - - batch_size, duration, height, width, _ = x.shape - pt, ph, pw = self.patch_size - return ( - x.view(batch_size, duration, height, width, -1, pt, ph, pw) - .permute(0, 1, 5, 2, 6, 3, 7, 4) - .flatten(1, 2) - .flatten(2, 3) - .flatten(3, 4) - ) - - class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): """Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head).""" @@ -179,39 +115,6 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): AttentionBackendEnum.TORCH_SDPA, } - @staticmethod - def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None: - if tp_size <= 0: - raise ValueError("Kandinsky6SR TP size must be positive.") - for name, value in ( - ("num_attention_heads", arch.num_attention_heads), - ("ff_dim", arch.ff_dim), - ): - if value % tp_size: - raise ValueError( - f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}." - ) - - @staticmethod - def _validate_sequence_parallel_config( - *, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int - ) -> None: - if ulysses_size <= 0: - raise ValueError("Kandinsky6SR Ulysses size must be positive.") - if ring_size <= 0: - raise ValueError("Kandinsky6SR ring size must be positive.") - if ulysses_size == 1 and ring_size == 1: - return - # Ring Attention rotates whole K/V shards between ranks rather than - # splitting heads, so only Ulysses constrains head divisibility. - local_heads = arch.num_attention_heads // tp_size - if local_heads % ulysses_size: - raise ValueError( - f"Kandinsky6SR TP-local attention heads {local_heads} must be " - f"divisible by Ulysses size {ulysses_size} (total heads=" - f"{arch.num_attention_heads}, TP={tp_size})." - ) - def __init__( self, config: Kandinsky6SRDitConfig, @@ -223,45 +126,19 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): self.quant_config = quant_config head_dim = sum(arch.axes_dims) - tp_size = get_tp_world_size() - ulysses_size, _ = get_ulysses_ctx() - ring_size, _ = get_ring_ctx() - self._validate_tp_config(arch=arch, tp_size=tp_size) - self._validate_sequence_parallel_config( - arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size - ) + _validate_parallelism(arch.num_attention_heads, ff_dim=arch.ff_dim) self.in_visual_dim = arch.in_visual_dim - self.base_out_visual_dim = arch.base_out_visual_dim - self.n_grid = arch.n_grid self.model_dim = arch.model_dim self.patch_size = arch.patch_size self.use_motion_score = arch.use_motion_score self.use_lq_noise_cond = arch.use_lq_noise_cond - # The input width is fixed by the *trained* configuration ... + # overrides change inference behavior, never the trained input width self.visual_embed_dim = ( 2 * arch.in_visual_dim + 1 if arch.trained_wide_input else arch.in_visual_dim ) - self._build_layers(arch, head_dim=head_dim, quant_config=quant_config) - - self.hidden_size = arch.hidden_size - self.num_attention_heads = arch.num_attention_heads - self.num_channels_latents = arch.num_channels_latents - self.layer_names = list(_BLOCK_CONTAINERS) - # ... while inference behaviour comes from the post-load overrides. - self._apply_attribute_overrides(arch) - self._require_all_checkpoint_keys() - self.__post_init__() - - def _build_layers( - self, - arch: Kandinsky6SRArchConfig, - *, - head_dim: int, - quant_config: QuantizationConfig | None, - ) -> None: self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim) if arch.use_motion_score: self.motion_embeddings = Kandinsky6TimeEmbeddings( @@ -295,35 +172,17 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): for i in range(arch.num_visual_blocks) ] ) - self.out_layer = Kandinsky6SROutLayer( + self.out_layer = Kandinsky6OutLayer( arch.model_dim, arch.time_dim, - arch.head_width, + arch.out_visual_dim, arch.patch_size, ) - def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None: - self.instruct_type = arch.instruct_type - self.visual_cond = arch.visual_cond - self.attention_params = arch.attention_params - for name in ATTRIBUTE_OVERRIDE_WHITELIST: - if name in arch.attribute_overrides: - setattr(self, name, arch.attribute_overrides[name]) - self._check_input_width_matches_overrides() - sparse_type = arch.requested_sparse_attention() - if sparse_type is not None: - logger.warning( - "Kandinsky6SR: attention_params request %r sparse attention; this port " - "runs dense attention, so outputs differ from the sparse reference.", - sparse_type, - ) - - def _check_input_width_matches_overrides(self) -> None: - run_wide = self.visual_cond or self.instruct_type in ( - "channel", - "hybrid", - "hybrid_anchor", - ) + self.instruct_type = arch.effective_override("instruct_type") + self.visual_cond = arch.effective_override("visual_cond") + self.attention_params = arch.effective_override("attention_params") + run_wide = self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim if run_width != self.visual_embed_dim: raise ValueError( @@ -334,10 +193,21 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): "under instruct_type='noise'." ) - def _require_all_checkpoint_keys(self) -> None: - """A missing tensor must fail loudly instead of being zero-filled by the loader.""" + sparse_type = arch.requested_sparse_attention() + if sparse_type is not None: + logger.warning( + "Kandinsky6SR: attention_params request %r sparse attention; this port " + "runs dense attention, so outputs differ from the sparse reference.", + sparse_type, + ) + self.hidden_size = arch.hidden_size + self.num_attention_heads = arch.num_attention_heads + self.num_channels_latents = arch.num_channels_latents + self.layer_names = list(_BLOCK_CONTAINERS) + # missing checkpoint tensors must fail, not be zero-filled for param in self.parameters(): param.missing_param_init = "error" + self.__post_init__() def preprocess_loaded_state_dict( self, weight_iterator: Iterable[tuple[str, torch.Tensor]] @@ -371,7 +241,16 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): timesteps are scaled by 1000. DX channels are ordered grid-major; the scheduler reshapes them into n_grid velocity predictions.""" - self._check_latent_shape(hidden_states) + if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim: + raise ValueError( + "Kandinsky6SR expects a [B, T, H, W, " + f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}" + ) + height, width = hidden_states.shape[2:4] + if height % self.patch_size[1] or width % self.patch_size[2]: + raise ValueError( + f"latent {height}x{width} is not divisible by patch {self.patch_size}" + ) compute_dtype = self.visual_embeddings.in_layer.weight.dtype time_embed = self.time_embeddings(timestep) if motion_score is not None and self.use_motion_score: @@ -397,19 +276,11 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): attn_mask_meta=attn_meta, ) visual_embed = gather_seq(visual_embed, shard.orig_len) - return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed) - - def _check_latent_shape(self, hidden_states: torch.Tensor) -> None: - if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim: - raise ValueError( - "Kandinsky6SR expects a [B, T, H, W, " - f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}" - ) - height, width = hidden_states.shape[2:4] - if height % self.patch_size[1] or width % self.patch_size[2]: - raise ValueError( - f"latent {height}x{width} is not divisible by patch {self.patch_size}" - ) + return self.out_layer( + visual_embed.reshape(*visual_shape, -1), + time_embed, + compute_dtype=self.out_layer.out_layer.weight.dtype, + ) def post_load_weights(self) -> None: """Rebuild RoPE / time-embedding tables that a meta-device init left on meta.""" diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py b/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py index 819b101770..b1fc5bd4af 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py @@ -25,119 +25,10 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler ) -class DXPolicy: - def __init__( # noqa: PLR0913 - self, - denoising_output: torch.Tensor, - x_t_src: torch.Tensor, - sigma_t_src: torch.Tensor, - segment_size: float | torch.Tensor = 1.0, - shift: float = 1.0, - mode: str = "grid", - eps: float = 1e-4, - ) -> None: - self.x_t_src = x_t_src - self.ndim = x_t_src.dim() - self.shift = shift - self.eps = eps - if mode not in ("grid", "polynomial"): - raise ValueError(f"Unknown mode: {mode}") - self.mode = mode - - self.sigma_t_src = sigma_t_src.reshape( - *sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1]) - ) - self.raw_t_src = self._unwarp_t(self.sigma_t_src) - segment = segment_size - if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim(): - segment = segment.reshape( - *segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1]) - ) - self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0) - self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps) - self.denoising_output_x_0 = self._u_to_x_0( - denoising_output, self.x_t_src, self.sigma_t_src - ) - - @staticmethod - def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: - n = x.size(1) - if n < 2: # noqa: PLR2004 - return x.squeeze(1) - t = t.clamp(min=0, max=1) * (n - 1) - t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2) - t1 = t0 + 1 - indices = torch.stack([t0, t1], dim=1) - values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:])) - return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1] - - def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor: - return sigma_t / (self.shift + (1 - self.shift) * sigma_t) - - @staticmethod - def _u_to_x_0( - denoising_output: torch.Tensor, - x_t: torch.Tensor, - sigma_t: torch.Tensor, - ) -> torch.Tensor: - return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output - - def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor: - sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1])) - raw_t = self._unwarp_t(sigma_t) - if self.mode == "grid": - x_0 = self._interpolate( - self.denoising_output_x_0, - (raw_t - self.raw_t_dst) / self.segment_size, - ) - else: - p_order = self.denoising_output_x_0.size(1) - diff_t = self.raw_t_src - raw_t - basis = torch.stack([diff_t**i for i in range(p_order)], dim=1) - x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1) - return (x_t - x_0) / sigma_t.clamp(min=self.eps) - - def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor: return shift * t / (1 + (shift - 1) * t) -def policy_rollout_fm( # noqa: PLR0913 - x_t_start: torch.Tensor, - sigma_t_start: torch.Tensor, - raw_t_start: torch.Tensor, - raw_t_end: torch.Tensor, - total_substeps: int, - policy: DXPolicy, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - num_batches = x_t_start.size(0) - ndim = x_t_start.dim() - shape = (num_batches, *((ndim - 1) * [1])) - raw_t_start = raw_t_start.reshape(shape) - raw_t_end = raw_t_end.reshape(shape) - sigma_t = sigma_t_start.reshape(shape) - - delta_raw_t = raw_t_start - raw_t_end - num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1) - substep_size = delta_raw_t / num_substeps - max_num_substeps = num_substeps.max() - - raw_t = raw_t_start - x_t = x_t_start - for substep_id in range(max_num_substeps.item()): - velocity = policy.pi(x_t, sigma_t) - raw_t_minus = (raw_t - substep_size).clamp(min=0) - sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift) - x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t) - - active_mask = num_substeps > substep_id - x_t = torch.where(active_mask, x_t_minus, x_t) - sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t) - raw_t = torch.where(active_mask, raw_t_minus, raw_t) - - return x_t, sigma_t, sigma_t.flatten() * 1_000 - - class PiflowScheduler(FlowMatchEulerDiscreteScheduler): is_piflow = True @@ -239,25 +130,41 @@ class PiflowScheduler(FlowMatchEulerDiscreteScheduler): sigma_src = self.sigmas[step_index].to(device=sample.device) token_shape = (sample.shape[0], *((sample.ndim - 1) * [1])) sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape) - segment = (raw_src - raw_dst).expand(sample.shape[0]) - policy = DXPolicy( - model_output, - sample, - sigma, - segment, - shift=float(self.config.shift), - mode="grid", - eps=self.eps, - ) - updated, _, _ = policy_rollout_fm( - sample, - sigma, - raw_src.expand(sample.shape[0]), - raw_dst.expand(sample.shape[0]), - self.num_policy_substeps, - policy, - ) - return updated.to(dtype=sample.dtype) + segment = (raw_src - raw_dst).expand(sample.shape[0]).reshape(token_shape) + shift = float(self.config.shift) + policy_src = sigma / (shift + (1 - shift) * sigma) + policy_dst = (policy_src - segment).clamp(min=0) + policy_segment = (policy_src - policy_dst).clamp(min=self.eps) + x0_grid = sample.unsqueeze(1) - sigma.unsqueeze(1) * model_output + + raw_t = raw_src.expand(sample.shape[0]).reshape(token_shape) + delta = raw_t - raw_dst.expand(sample.shape[0]).reshape(token_shape) + substeps = (delta * self.num_policy_substeps).round().long().clamp(min=1) + substep_size = delta / substeps + for index in range(substeps.max().item()): + # the checkpoint predicts x0 on a grid in unwarped time + policy_t = sigma / (shift + (1 - shift) * sigma) + t = ((policy_t - policy_dst) / policy_segment).clamp(0, 1) * ( + self.n_grid - 1 + ) + t0 = t.floor().long().clamp(0, self.n_grid - 2) + t1 = t0 + 1 + indices = torch.stack([t0, t1], dim=1) + values = torch.gather( + x0_grid, 1, indices.expand(-1, -1, *x0_grid.shape[2:]) + ) + x0 = (t1 - t) * values[:, 0] + (t - t0) * values[:, 1] + velocity = (sample - x0) / sigma.clamp(min=self.eps) + # release grid interpolation temporaries before the state update/next gather + del values, x0, indices, t, t0, t1, policy_t + next_raw = (raw_t - substep_size).clamp(min=0) + next_sigma = shift_timesteps(next_raw, shift) + updated = sample + velocity * (next_sigma - sigma) + active = substeps > index + sample = torch.where(active, updated, sample) + sigma = torch.where(active, next_sigma, sigma) + raw_t = torch.where(active, next_raw, raw_t) + return sample def _step_index_for(self, timestep: torch.Tensor | float) -> int: if self.step_index is None: diff --git a/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py b/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py index bbaed76bb3..df9d818995 100644 --- a/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py +++ b/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py @@ -48,9 +48,7 @@ class RMSNorm(nn.Module): self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1)) def forward(self, x: Tensor) -> Tensor: - # Upcast to float32 around the normalize (matches the Diffusers reference's - # Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the - # reference across the many ResidualBlock/X2Branch calls that route through this norm. + # preserve the reference's fp32 reduction followed by bf16 scale/gain normalized = F.normalize(x.float(), dim=1).to(x.dtype) return normalized * self.scale * self.gamma @@ -96,31 +94,8 @@ class ResidualBlock(nn.Module): return self.shortcut(x) + h -def nearest_2x(x: Tensor) -> Tensor: - """Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch).""" - b, c, t, h, w = x.shape - x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) - x = F.interpolate(x, scale_factor=2, mode="nearest") - return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4) - - -class PXSUpsample(nn.Module): - """2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``.""" - - def __init__(self, channels: int) -> None: - super().__init__() - self.spatial_conv = nn.Conv3d( - channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1) - ) - self.linear = nn.Conv3d(channels, channels, kernel_size=1) - - def forward(self, x: Tensor) -> Tensor: - up = nearest_2x(x) - return self.linear(up + self.spatial_conv(up)) - - class X2Finisher(nn.Module): - """The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize).""" + """Residual spatial convolution and channel projection, without resizing.""" def __init__(self, channels: int) -> None: super().__init__() @@ -133,19 +108,21 @@ class X2Finisher(nn.Module): return self.linear(x + self.spatial_conv(x)) +class PXSUpsample(X2Finisher): + """Apply the same projection after nearest-neighbor 2x spatial resizing.""" + + def forward(self, x: Tensor) -> Tensor: + b, c, t, h, w = x.shape + x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + x = F.interpolate(x, scale_factor=2, mode="nearest") + x = x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4) + return super().forward(x) + + def _stem(in_channels: int, out_channels: int) -> nn.Sequential: return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3)) -def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential: - """Unused training head whose checkpoint layout remains positional.""" - return nn.Sequential( - RMSNorm(channels), - nn.SiLU(), - ReplicateTimeConv3d(channels, out_channels, kernel_size=3), - ) - - class Kandinsky6SRLatentUpscalerOutputHead(nn.Module): """Named ``norm -> activation -> conv`` projection used by current checkpoints.""" @@ -182,18 +159,6 @@ def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor: return x -def _second_stage( - upsample: nn.Module, - blocks: nn.Sequential, - head: Kandinsky6SRLatentUpscalerOutputHead, - x: Tensor, - zq: Tensor, -) -> Tensor: - """``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``.""" - x = _run_blocks(blocks, upsample(x), zq) - return head(x, zq) - - class X2Branch(nn.Module): """Weights exclusive to the x2 path.""" @@ -213,7 +178,8 @@ class X2Branch(nn.Module): x = _run_blocks(self.adapter, x, zq) x = self.finisher(x) x = _run_blocks(self.mid_blocks, x, zq) - return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq) + x = _run_blocks(self.blocks, self.upsample(x), zq) + return self.output_proj(x, zq) class Kandinsky6SRLatentUpscaler(nn.Module): @@ -227,7 +193,9 @@ class Kandinsky6SRLatentUpscaler(nn.Module): er = config.expand_ratio self.input_proj = _stem(c, w1) # The 2x deep-supervision head of training: part of the checkpoint, never run at inference. - self.mid_output_head = _mid_output_head(w2, c) + self.mid_output_head = nn.Sequential( + RMSNorm(w2), nn.SiLU(), ReplicateTimeConv3d(w2, c, kernel_size=3) + ) self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c) self.upsample_1 = PXSUpsample(w1) self.upsample_2 = PXSUpsample(w2) @@ -254,7 +222,8 @@ class Kandinsky6SRLatentUpscaler(nn.Module): ) x = _run_blocks(self.pre_blocks, self.input_proj(z), zq) x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq) - return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq) + x = _run_blocks(self.post_blocks, self.upsample_2(x), zq) + return self.output_proj(x, zq) class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin): diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py index aa8ed2bfb3..064dd01dd1 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py @@ -48,24 +48,13 @@ class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin): # audio latent stride; frame-count alignment is owned by the pipeline config self.downsample_factor = 1024 - self.mel_converter = MelConverter( - sampling_rate=44100, - n_fft=2048, - num_mels=128, - hop_size=512, - win_size=2048, - fmin=0, - fmax=22050, - ) + self.mel_converter = MelConverter() # checkpoint weights include the encoder regardless of need_vae_encoder self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True) # normalize random initialization before loading; checkpoint weights are # already normalized and must never be renormalized self.vae.remove_weight_norm() self.vocoder = BigVGANV2(dict(arch.vocoder_config)) - # strip PyTorch weight_norm before loading plain .weight checkpoint keys; - # this differs from MMAudio's RMS renormalization above - self.vocoder.remove_weight_norm() # resolution containers are not called; hooks belong on their blocks self.layer_names = ( diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py index cab3d2ff12..39bf72ccdb 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py @@ -1,350 +1,141 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2024 NVIDIA CORPORATION. -# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's -# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is -# intentionally excluded; this module uses the portable PyTorch path for -# deterministic loading and parity across platforms. +# Adapted from NVIDIA BigVGAN-v2 (MIT) via FastVideo. # Original license terms are retained in THIRD_PARTY_LICENSES. -# -# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/ -# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than -# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` -- -# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and -# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more), -# while this implementation supports either resblock variant or activation. The -# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/ -# Activation1d) ARE architecture-agnostic DSP building blocks shared with -# that same sibling module, though -- both import ``Activation1d`` from -# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside -# ``common.py``) instead of each keeping its own copy. - -from __future__ import annotations +"""BigVGAN-v2 with the plain weights stored in Kandinsky audio checkpoints.""" +from types import SimpleNamespace from typing import Any import torch -import torch.nn as nn -from torch.nn import Conv1d, ConvTranspose1d -from torch.nn.utils.parametrizations import weight_norm -from torch.nn.utils.parametrize import remove_parametrizations +from torch import nn from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d -class AttrDict(dict): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.__dict__ = self - - -def get_padding(kernel_size: int, dilation: int = 1) -> int: - return int((kernel_size * dilation - dilation) / 2) - - -def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None: - if "Conv" in module.__class__.__name__: - module.weight.data.normal_(mean, std) - - class Snake(nn.Module): - def __init__( - self, - in_features: int, - alpha: float = 1.0, - alpha_trainable: bool = True, - alpha_logscale: bool = False, - ) -> None: + def __init__(self, channels: int, *, logscale: bool, separate_beta: bool): super().__init__() - self.in_features = in_features - self.alpha_logscale = alpha_logscale - initial = ( - torch.zeros(in_features) * alpha - if alpha_logscale - else torch.ones(in_features) * alpha - ) - self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable) - self.no_div_by_zero = 1e-9 + initial = torch.zeros(channels) if logscale else torch.ones(channels) + self.alpha = nn.Parameter(initial) + self.beta = nn.Parameter(initial.clone()) if separate_beta else None + self.logscale = logscale def forward(self, x: torch.Tensor) -> torch.Tensor: - alpha = self.alpha.unsqueeze(0).unsqueeze(-1) - if self.alpha_logscale: - alpha = torch.exp(alpha) - return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow( - torch.sin(x * alpha), 2 + alpha = self.alpha[None, :, None] + beta = alpha if self.beta is None else self.beta[None, :, None] + if self.logscale: + alpha = alpha.exp() + beta = alpha if self.beta is None else beta.exp() + return x + (1.0 / (beta + 1e-9)) * torch.sin(x * alpha).pow(2) + + +def _activation(config: SimpleNamespace, channels: int) -> Activation1d: + if config.activation not in ("snake", "snakebeta"): + raise ValueError(f"Unsupported BigVGAN activation: {config.activation}") + return Activation1d( + Snake( + channels, + logscale=config.snake_logscale, + separate_beta=config.activation == "snakebeta", ) + ) -class SnakeBeta(nn.Module): - def __init__( - self, - in_features: int, - alpha: float = 1.0, - alpha_trainable: bool = True, - alpha_logscale: bool = False, - ) -> None: +class AMPBlock(nn.Module): + def __init__(self, config, channels: int, kernel: int, dilations): super().__init__() - self.in_features = in_features - self.alpha_logscale = alpha_logscale - initial = ( - torch.zeros(in_features) * alpha - if alpha_logscale - else torch.ones(in_features) * alpha + self.paired = config.resblock == "1" + convs = nn.ModuleList( + nn.Conv1d( + channels, + channels, + kernel, + dilation=rate, + padding=(kernel - 1) * rate // 2, + ) + for rate in dilations ) - self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable) - self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable) - self.no_div_by_zero = 1e-9 - - def forward(self, x: torch.Tensor) -> torch.Tensor: - alpha = self.alpha.unsqueeze(0).unsqueeze(-1) - beta = self.beta.unsqueeze(0).unsqueeze(-1) - if self.alpha_logscale: - alpha = torch.exp(alpha) - beta = torch.exp(beta) - return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow( - torch.sin(x * alpha), 2 - ) - - -def _activation(name: str, channels: int, logscale: bool) -> Activation1d: - if name == "snake": - activation = Snake(channels, alpha_logscale=logscale) - elif name == "snakebeta": - activation = SnakeBeta(channels, alpha_logscale=logscale) - else: - raise ValueError(f"Unsupported BigVGAN activation: {name}") - return Activation1d(activation) - - -class AMPBlock1(nn.Module): - def __init__( - self, - config: AttrDict, - channels: int, - kernel_size: int = 3, - dilation: tuple[int, ...] = (1, 3, 5), - activation: str = "snake", - ) -> None: - super().__init__() - self.convs1 = nn.ModuleList( - [ - weight_norm( - Conv1d( - channels, - channels, - kernel_size, - stride=1, - dilation=rate, - padding=get_padding(kernel_size, rate), - ) - ) - for rate in dilation - ] - ) - self.convs1.apply(init_weights) - self.convs2 = nn.ModuleList( - [ - weight_norm( - Conv1d( - channels, - channels, - kernel_size, - stride=1, - dilation=1, - padding=get_padding(kernel_size, 1), - ) - ) - for _ in dilation - ] - ) - self.convs2.apply(init_weights) + # retain both published state-dict layouts + if self.paired: + self.convs1 = convs + self.convs2 = nn.ModuleList( + nn.Conv1d(channels, channels, kernel, padding=(kernel - 1) // 2) + for _ in dilations + ) + else: + self.convs = convs self.activations = nn.ModuleList( - [ - _activation(activation, channels, config.snake_logscale) - for _ in range(len(self.convs1) + len(self.convs2)) - ] + _activation(config, channels) + for _ in range(len(dilations) * (2 if self.paired else 1)) ) def forward(self, x: torch.Tensor) -> torch.Tensor: - acts1, acts2 = self.activations[::2], self.activations[1::2] - for conv1, conv2, act1, act2 in zip( - self.convs1, self.convs2, acts1, acts2, strict=True - ): - residual = conv2(act2(conv1(act1(x)))) - x = residual + x + if self.paired: + for index, (conv1, conv2) in enumerate(zip(self.convs1, self.convs2)): + hidden = conv1(self.activations[2 * index](x)) + x = conv2(self.activations[2 * index + 1](hidden)) + x + else: + for conv, activation in zip(self.convs, self.activations): + x = conv(activation(x)) + x return x - def remove_weight_norm(self) -> None: - for layer in self.convs1: - remove_parametrizations(layer, "weight") - for layer in self.convs2: - remove_parametrizations(layer, "weight") - - -class AMPBlock2(nn.Module): - def __init__( - self, - config: AttrDict, - channels: int, - kernel_size: int = 3, - dilation: tuple[int, ...] = (1, 3, 5), - activation: str = "snake", - ) -> None: - super().__init__() - self.convs = nn.ModuleList( - [ - weight_norm( - Conv1d( - channels, - channels, - kernel_size, - stride=1, - dilation=rate, - padding=get_padding(kernel_size, rate), - ) - ) - for rate in dilation - ] - ) - self.convs.apply(init_weights) - self.activations = nn.ModuleList( - [ - _activation(activation, channels, config.snake_logscale) - for _ in self.convs - ] - ) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - for conv, activation in zip(self.convs, self.activations, strict=True): - x = conv(activation(x)) + x - return x - - def remove_weight_norm(self) -> None: - for layer in self.convs: - remove_parametrizations(layer, "weight") - class BigVGANV2(nn.Module): - """BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path.""" - def __init__(self, config: dict[str, Any]) -> None: super().__init__() - config = dict(config) - config.pop("_class_name", None) - config.pop("architectures", None) - weight_norm_removed = bool(config.pop("weight_norm_removed", False)) - self.config = AttrDict(config) - if self.config.get("use_cuda_kernel", False): + if config.get("use_cuda_kernel", False): raise ValueError( "Kandinsky6 BigVGANV2 supports only the portable PyTorch path" ) - self.config["use_cuda_kernel"] = False - self.num_kernels = len(self.config.resblock_kernel_sizes) - self.num_upsamples = len(self.config.upsample_rates) - self.conv_pre = weight_norm( - Conv1d( - self.config.num_mels, - self.config.upsample_initial_channel, - 7, - 1, - padding=3, - ) - ) - if self.config.resblock == "1": - block_class = AMPBlock1 - elif self.config.resblock == "2": - block_class = AMPBlock2 - else: - raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}") - + h = SimpleNamespace(**config) + if h.resblock not in ("1", "2"): + raise ValueError(f"Unsupported BigVGAN resblock: {h.resblock}") + self.num_kernels = len(h.resblock_kernel_sizes) + self.conv_pre = nn.Conv1d(h.num_mels, h.upsample_initial_channel, 7, padding=3) self.ups = nn.ModuleList() - for index, (rate, kernel) in enumerate( - zip( - self.config.upsample_rates, - self.config.upsample_kernel_sizes, - strict=True, - ) - ): + self.resblocks = nn.ModuleList() + channels = h.upsample_initial_channel + for rate, kernel in zip(h.upsample_rates, h.upsample_kernel_sizes, strict=True): self.ups.append( nn.ModuleList( [ - weight_norm( - ConvTranspose1d( - self.config.upsample_initial_channel // (2**index), - self.config.upsample_initial_channel - // (2 ** (index + 1)), - kernel, - rate, - padding=(kernel - rate) // 2, - ) + nn.ConvTranspose1d( + channels, + channels // 2, + kernel, + rate, + padding=(kernel - rate) // 2, ) ] ) ) - - self.resblocks = nn.ModuleList() - for index in range(len(self.ups)): - channels = self.config.upsample_initial_channel // (2 ** (index + 1)) - for kernel, dilation in zip( - self.config.resblock_kernel_sizes, - self.config.resblock_dilation_sizes, - strict=True, - ): - self.resblocks.append( - block_class( - self.config, - channels, - kernel, - tuple(dilation), - activation=self.config.activation, - ) + channels //= 2 + self.resblocks.extend( + AMPBlock(h, channels, kernel, dilations) + for kernel, dilations in zip( + h.resblock_kernel_sizes, h.resblock_dilation_sizes, strict=True ) - - channels = self.config.upsample_initial_channel // (2 ** len(self.ups)) - self.activation_post = _activation( - self.config.activation, channels, self.config.snake_logscale + ) + self.activation_post = _activation(h, channels) + self.conv_post = nn.Conv1d( + channels, 1, 7, padding=3, bias=config.get("use_bias_at_final", True) ) - self.use_bias_at_final = self.config.get("use_bias_at_final", True) - self.conv_post = weight_norm( - Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final) - ) - for upsampler in self.ups: - upsampler.apply(init_weights) - self.conv_post.apply(init_weights) - self.use_tanh_at_final = self.config.get("use_tanh_at_final", True) - if weight_norm_removed: - self.remove_weight_norm() + self.use_tanh_at_final = config.get("use_tanh_at_final", True) def forward(self, mel: torch.Tensor) -> torch.Tensor: hidden = self.conv_pre(mel) - for index in range(self.num_upsamples): - for upsampler in self.ups[index]: + for index, upsamplers in enumerate(self.ups): + for upsampler in upsamplers: hidden = upsampler(hidden) accumulated = None for kernel in range(self.num_kernels): - block_output = self.resblocks[index * self.num_kernels + kernel](hidden) + output = self.resblocks[index * self.num_kernels + kernel](hidden) if accumulated is None: - accumulated = block_output + accumulated = output else: - # Preserve BigVGAN's published sequential accumulation - # order. Tree reduction drifts through later nonlinear - # upsampling stages with the full checkpoint. - accumulated += block_output - assert accumulated is not None + # preserve sequential accumulation through nonlinear upsampling + accumulated += output hidden = accumulated / self.num_kernels hidden = self.conv_post(self.activation_post(hidden)) - if self.use_tanh_at_final: - return torch.tanh(hidden) - return torch.clamp(hidden, min=-1.0, max=1.0) - - def remove_weight_norm(self) -> None: - try: - for upsamplers in self.ups: - for upsampler in upsamplers: - remove_parametrizations(upsampler, "weight") - for block in self.resblocks: - block.remove_weight_norm() - remove_parametrizations(self.conv_pre, "weight") - remove_parametrizations(self.conv_post, "weight") - except ValueError: - # Idempotent for pipeline setup and converted checkpoints. - return + return torch.tanh(hidden) if self.use_tanh_at_final else hidden.clamp(-1.0, 1.0) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py index 17276fa45a..5f1068e3df 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py @@ -1,25 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # # Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License, -# Copyright (c) 2024 Sony Research Inc., via FastVideo's -# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram -# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae`` -# (see kandinsky6_audio.py in the parent directory). +# Copyright (c) 2024 Sony Research Inc., via FastVideo. # Original license terms are retained in THIRD_PARTY_LICENSES. -# -# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its -# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is -# dropped -- sglang's generic VAELoader never calls a per-submodule -# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict`` -# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code -# here. """Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec).""" from __future__ import annotations import logging import math -from typing import Any +from collections import OrderedDict import torch import torch.nn as nn @@ -29,435 +19,6 @@ from einops import rearrange logger = logging.getLogger(__name__) -DATA_MEAN_80D = [ - -1.6058, - -1.3676, - -1.2520, - -1.2453, - -1.2078, - -1.2224, - -1.2419, - -1.2439, - -1.2922, - -1.2927, - -1.3170, - -1.3543, - -1.3401, - -1.3836, - -1.3907, - -1.3912, - -1.4313, - -1.4152, - -1.4527, - -1.4728, - -1.4568, - -1.5101, - -1.5051, - -1.5172, - -1.5623, - -1.5373, - -1.5746, - -1.5687, - -1.6032, - -1.6131, - -1.6081, - -1.6331, - -1.6489, - -1.6489, - -1.6700, - -1.6738, - -1.6953, - -1.6969, - -1.7048, - -1.7280, - -1.7361, - -1.7495, - -1.7658, - -1.7814, - -1.7889, - -1.8064, - -1.8221, - -1.8377, - -1.8417, - -1.8643, - -1.8857, - -1.8929, - -1.9173, - -1.9379, - -1.9531, - -1.9673, - -1.9824, - -2.0042, - -2.0215, - -2.0436, - -2.0766, - -2.1064, - -2.1418, - -2.1855, - -2.2319, - -2.2767, - -2.3161, - -2.3572, - -2.3954, - -2.4282, - -2.4659, - -2.5072, - -2.5552, - -2.6074, - -2.6584, - -2.7107, - -2.7634, - -2.8266, - -2.8981, - -2.9673, -] - -DATA_STD_80D = [ - 1.0291, - 1.0411, - 1.0043, - 0.9820, - 0.9677, - 0.9543, - 0.9450, - 0.9392, - 0.9343, - 0.9297, - 0.9276, - 0.9263, - 0.9242, - 0.9254, - 0.9232, - 0.9281, - 0.9263, - 0.9315, - 0.9274, - 0.9247, - 0.9277, - 0.9199, - 0.9188, - 0.9194, - 0.9160, - 0.9161, - 0.9146, - 0.9161, - 0.9100, - 0.9095, - 0.9145, - 0.9076, - 0.9066, - 0.9095, - 0.9032, - 0.9043, - 0.9038, - 0.9011, - 0.9019, - 0.9010, - 0.8984, - 0.8983, - 0.8986, - 0.8961, - 0.8962, - 0.8978, - 0.8962, - 0.8973, - 0.8993, - 0.8976, - 0.8995, - 0.9016, - 0.8982, - 0.8972, - 0.8974, - 0.8949, - 0.8940, - 0.8947, - 0.8936, - 0.8939, - 0.8951, - 0.8956, - 0.9017, - 0.9167, - 0.9436, - 0.9690, - 1.0003, - 1.0225, - 1.0381, - 1.0491, - 1.0545, - 1.0604, - 1.0761, - 1.0929, - 1.1089, - 1.1196, - 1.1176, - 1.1156, - 1.1117, - 1.1070, -] - -DATA_MEAN_128D = [ - -3.3462, - -2.6723, - -2.4893, - -2.3143, - -2.2664, - -2.3317, - -2.1802, - -2.4006, - -2.2357, - -2.4597, - -2.3717, - -2.4690, - -2.5142, - -2.4919, - -2.6610, - -2.5047, - -2.7483, - -2.5926, - -2.7462, - -2.7033, - -2.7386, - -2.8112, - -2.7502, - -2.9594, - -2.7473, - -3.0035, - -2.8891, - -2.9922, - -2.9856, - -3.0157, - -3.1191, - -2.9893, - -3.1718, - -3.0745, - -3.1879, - -3.2310, - -3.1424, - -3.2296, - -3.2791, - -3.2782, - -3.2756, - -3.3134, - -3.3509, - -3.3750, - -3.3951, - -3.3698, - -3.4505, - -3.4509, - -3.5089, - -3.4647, - -3.5536, - -3.5788, - -3.5867, - -3.6036, - -3.6400, - -3.6747, - -3.7072, - -3.7279, - -3.7283, - -3.7795, - -3.8259, - -3.8447, - -3.8663, - -3.9182, - -3.9605, - -3.9861, - -4.0105, - -4.0373, - -4.0762, - -4.1121, - -4.1488, - -4.1874, - -4.2461, - -4.3170, - -4.3639, - -4.4452, - -4.5282, - -4.6297, - -4.7019, - -4.7960, - -4.8700, - -4.9507, - -5.0303, - -5.0866, - -5.1634, - -5.2342, - -5.3242, - -5.4053, - -5.4927, - -5.5712, - -5.6464, - -5.7052, - -5.7619, - -5.8410, - -5.9188, - -6.0103, - -6.0955, - -6.1673, - -6.2362, - -6.3120, - -6.3926, - -6.4797, - -6.5565, - -6.6511, - -6.8130, - -6.9961, - -7.1275, - -7.2457, - -7.3576, - -7.4663, - -7.6136, - -7.7469, - -7.8815, - -8.0132, - -8.1515, - -8.3071, - -8.4722, - -8.7418, - -9.3975, - -9.6628, - -9.7671, - -9.8863, - -9.9992, - -10.0860, - -10.1709, - -10.5418, - -11.2795, - -11.3861, -] - -DATA_STD_128D = [ - 2.3804, - 2.4368, - 2.3772, - 2.3145, - 2.2803, - 2.2510, - 2.2316, - 2.2083, - 2.1996, - 2.1835, - 2.1769, - 2.1659, - 2.1631, - 2.1618, - 2.1540, - 2.1606, - 2.1571, - 2.1567, - 2.1612, - 2.1579, - 2.1679, - 2.1683, - 2.1634, - 2.1557, - 2.1668, - 2.1518, - 2.1415, - 2.1449, - 2.1406, - 2.1350, - 2.1313, - 2.1415, - 2.1281, - 2.1352, - 2.1219, - 2.1182, - 2.1327, - 2.1195, - 2.1137, - 2.1080, - 2.1179, - 2.1036, - 2.1087, - 2.1036, - 2.1015, - 2.1068, - 2.0975, - 2.0991, - 2.0902, - 2.1015, - 2.0857, - 2.0920, - 2.0893, - 2.0897, - 2.0910, - 2.0881, - 2.0925, - 2.0873, - 2.0960, - 2.0900, - 2.0957, - 2.0958, - 2.0978, - 2.0936, - 2.0886, - 2.0905, - 2.0845, - 2.0855, - 2.0796, - 2.0840, - 2.0813, - 2.0817, - 2.0838, - 2.0840, - 2.0917, - 2.1061, - 2.1431, - 2.1976, - 2.2482, - 2.3055, - 2.3700, - 2.4088, - 2.4372, - 2.4609, - 2.4731, - 2.4847, - 2.5072, - 2.5451, - 2.5772, - 2.6147, - 2.6529, - 2.6596, - 2.6645, - 2.6726, - 2.6803, - 2.6812, - 2.6899, - 2.6916, - 2.6931, - 2.6998, - 2.7062, - 2.7262, - 2.7222, - 2.7158, - 2.7041, - 2.7485, - 2.7491, - 2.7451, - 2.7485, - 2.7233, - 2.7297, - 2.7233, - 2.7145, - 2.6958, - 2.6788, - 2.6439, - 2.6007, - 2.4786, - 2.2469, - 2.1877, - 2.1392, - 2.0717, - 2.0107, - 1.9676, - 1.9140, - 1.7102, - 0.9101, - 0.7164, -] - - _NORMALIZATION_EPSILON = 1e-4 _SILU_DIVISOR = 0.596 _RESIDUAL_BLEND = 0.3 @@ -516,33 +77,22 @@ class ResnetBlock1D(nn.Module): *, in_dim: int, out_dim: int | None = None, - conv_shortcut: bool = False, kernel_size: int = 3, - use_norm: bool = True, ) -> None: super().__init__() self.in_dim = in_dim self.out_dim = in_dim if out_dim is None else out_dim - self.use_conv_shortcut = conv_shortcut - self.use_norm = use_norm self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size) self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size) if self.in_dim != self.out_dim: - if conv_shortcut: - self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size) - else: - self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1) + self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1) def forward(self, x: torch.Tensor) -> torch.Tensor: - if self.use_norm: - x = _rms_normalize(x, dim=1) + x = _rms_normalize(x, dim=1) hidden = self.conv1(F.silu(x) / _SILU_DIVISOR) hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR) if self.in_dim != self.out_dim: - shortcut = ( - self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut - ) - x = shortcut(x) + x = self.nin_shortcut(x) return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR @@ -566,30 +116,60 @@ class AttnBlock1D(nn.Module): class Upsample1D(nn.Module): - def __init__(self, in_channels: int, with_conv: bool) -> None: + def __init__(self, in_channels: int) -> None: super().__init__() - self.with_conv = with_conv - if with_conv: - self.conv = _conv1d(in_channels, in_channels, 3) + self.conv = _conv1d(in_channels, in_channels, 3) def forward(self, x: torch.Tensor) -> torch.Tensor: x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact") - return self.conv(x) if self.with_conv else x + return self.conv(x) class Downsample1D(nn.Module): - def __init__(self, in_channels: int, with_conv: bool) -> None: + def __init__(self, in_channels: int) -> None: super().__init__() - self.with_conv = with_conv - if with_conv: - self.conv1 = _conv1d(in_channels, in_channels, 1) - self.conv2 = _conv1d(in_channels, in_channels, 1) + self.conv1 = _conv1d(in_channels, in_channels, 1) + self.conv2 = _conv1d(in_channels, in_channels, 1) def forward(self, x: torch.Tensor) -> torch.Tensor: - if self.with_conv: - x = self.conv1(x) - x = F.avg_pool1d(x, kernel_size=2, stride=2) - return self.conv2(x) if self.with_conv else x + return self.conv2(F.avg_pool1d(self.conv1(x), kernel_size=2, stride=2)) + + +def _mid_block(channels: int, kernel_size: int = 3) -> nn.Sequential: + return nn.Sequential( + OrderedDict( + block_1=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size), + attn_1=AttnBlock1D(channels), + block_2=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size), + ) + ) + + +class ResolutionBlock1D(nn.Module): + """Residual/attention stack with checkpoint-compatible block and attn names.""" + + def __init__(self, in_dim, out_dim, count, attention, kernel_size=3): + super().__init__() + self.block = nn.ModuleList() + self.attn = nn.ModuleList() + for index in range(count): + self.block.append( + ResnetBlock1D( + in_dim=in_dim if index == 0 else out_dim, + out_dim=out_dim, + kernel_size=kernel_size, + ) + ) + if attention: + self.attn.append(AttnBlock1D(out_dim)) + + def forward(self, x, clip_act): + for index, block in enumerate(self.block): + x = block(x) + if self.attn: + x = self.attn[index](x) + x = x.clamp(-clip_act, clip_act) + return x class Encoder1D(nn.Module): @@ -603,73 +183,38 @@ class Encoder1D(nn.Module): down_layers: list[int], in_dim: int, embed_dim: int, - resamp_with_conv: bool = True, - double_z: bool = True, kernel_size: int = 3, clip_act: float = 256.0, ) -> None: super().__init__() - self.dim = dim self.num_layers = len(ch_mult) - self.num_res_blocks = num_res_blocks - self.in_channels = in_dim self.clip_act = clip_act self.down_layers = down_layers - self.attn_layers = attn_layers self.conv_in = _conv1d(in_dim, dim, kernel_size) in_ch_mult = (1,) + ch_mult - self.in_ch_mult = in_ch_mult self.down = nn.ModuleList() for level in range(self.num_layers): - block = nn.ModuleList() - attn = nn.ModuleList() block_in = dim * in_ch_mult[level] block_out = dim * ch_mult[level] - for _ in range(num_res_blocks): - block.append( - ResnetBlock1D( - in_dim=block_in, - out_dim=block_out, - kernel_size=kernel_size, - use_norm=True, - ) - ) - block_in = block_out - if level in attn_layers: - attn.append(AttnBlock1D(block_in)) - down = nn.Module() - down.block = block - down.attn = attn + down = ResolutionBlock1D( + block_in, block_out, num_res_blocks, level in attn_layers, kernel_size + ) if level in down_layers: - down.downsample = Downsample1D(block_in, resamp_with_conv) + down.downsample = Downsample1D(block_out) self.down.append(down) - self.mid = nn.Module() - self.mid.block_1 = ResnetBlock1D( - in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True - ) - self.mid.attn_1 = AttnBlock1D(block_in) - self.mid.block_2 = ResnetBlock1D( - in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True - ) - output_dim = 2 * embed_dim if double_z else embed_dim - self.conv_out = _conv1d(block_in, output_dim, kernel_size) + self.mid = _mid_block(block_out, kernel_size) + self.conv_out = _conv1d(block_out, 2 * embed_dim, kernel_size) self.learnable_gain = nn.Parameter(torch.zeros([])) def forward(self, x: torch.Tensor) -> torch.Tensor: - states = [self.conv_in(x)] - for level in range(self.num_layers): - for block_index in range(self.num_res_blocks): - hidden = self.down[level].block[block_index](states[-1]) - if len(self.down[level].attn) > 0: - hidden = self.down[level].attn[block_index](hidden) - states.append(hidden.clamp(-self.clip_act, self.clip_act)) + hidden = self.conv_in(x) + for level, down in enumerate(self.down): + hidden = down(hidden, self.clip_act) if level in self.down_layers: - states.append(self.down[level].downsample(states[-1])) - hidden = self.mid.block_1(states[-1]) - hidden = self.mid.attn_1(hidden) - hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act) + hidden = down.downsample(hidden) + hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act) return _conv1d_with_gain( self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1 ) @@ -685,47 +230,27 @@ class Decoder1D(nn.Module): num_res_blocks: int, attn_layers: list[int], down_layers: list[int], - in_dim: int, embed_dim: int, kernel_size: int = 3, - resamp_with_conv: bool = True, clip_act: float = 256.0, ) -> None: super().__init__() - self.ch = dim self.num_layers = len(ch_mult) - self.num_res_blocks = num_res_blocks - self.in_channels = in_dim self.clip_act = clip_act self.down_layers = [level + 1 for level in down_layers] block_in = dim * ch_mult[-1] self.conv_in = _conv1d(embed_dim, block_in, kernel_size) - self.mid = nn.Module() - self.mid.block_1 = ResnetBlock1D( - in_dim=block_in, out_dim=block_in, use_norm=True - ) - self.mid.attn_1 = AttnBlock1D(block_in) - self.mid.block_2 = ResnetBlock1D( - in_dim=block_in, out_dim=block_in, use_norm=True - ) + self.mid = _mid_block(block_in) self.up = nn.ModuleList() for level in reversed(range(self.num_layers)): - block = nn.ModuleList() - attn = nn.ModuleList() block_out = dim * ch_mult[level] - for _ in range(num_res_blocks + 1): - block.append( - ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True) - ) - block_in = block_out - if level in attn_layers: - attn.append(AttnBlock1D(block_in)) - up = nn.Module() - up.block = block - up.attn = attn + up = ResolutionBlock1D( + block_in, block_out, num_res_blocks + 1, level in attn_layers + ) + block_in = block_out if level in self.down_layers: - up.upsample = Upsample1D(block_in, resamp_with_conv) + up.upsample = Upsample1D(block_in) self.up.insert(0, up) self.conv_out = _conv1d(block_in, out_dim, kernel_size) @@ -733,15 +258,9 @@ class Decoder1D(nn.Module): def forward(self, z: torch.Tensor) -> torch.Tensor: hidden = self.conv_in(z) - hidden = self.mid.block_1(hidden) - hidden = self.mid.attn_1(hidden) - hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act) + hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act) for level in reversed(range(self.num_layers)): - for block_index in range(self.num_res_blocks + 1): - hidden = self.up[level].block[block_index](hidden) - if len(self.up[level].attn) > 0: - hidden = self.up[level].attn[block_index](hidden) - hidden = hidden.clamp(-self.clip_act, self.clip_act) + hidden = self.up[level](hidden, self.clip_act) if level in self.down_layers: hidden = self.up[level].upsample(hidden) return _conv1d_with_gain( @@ -752,32 +271,26 @@ class Decoder1D(nn.Module): class MMAudioVAE(nn.Module): """MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio.""" - def __init__( - self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False - ) -> None: + def __init__(self, mode: str = "44k", need_encoder: bool = False) -> None: super().__init__() - if isinstance(mode, dict): - config = mode - mode = config.get("mode", "44k") - need_encoder = config.get("need_encoder", need_encoder) if mode == "16k": data_dim, embed_dim, hidden_dim = 80, 20, 384 - data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D elif mode == "44k": data_dim, embed_dim, hidden_dim = 128, 40, 512 - data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D else: raise ValueError(f"Unknown MMAudio VAE mode: {mode}") self.mode = mode self.embed_dim = embed_dim self._weights_normalized = False + # checkpoint buffers are required by native VAELoader's strict load self.register_buffer( - "data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1) + "data_mean", torch.zeros(1, data_dim, 1, dtype=torch.float32) ) self.register_buffer( - "data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1) + "data_std", torch.ones(1, data_dim, 1, dtype=torch.float32) ) + self.encoder = None if need_encoder: self.encoder = Encoder1D( dim=hidden_dim, @@ -794,7 +307,6 @@ class MMAudioVAE(nn.Module): num_res_blocks=2, attn_layers=[3], down_layers=[0], - in_dim=data_dim, out_dim=data_dim, embed_dim=embed_dim, ) @@ -803,7 +315,7 @@ class MMAudioVAE(nn.Module): self, mel: torch.Tensor, normalize_input: bool = True ) -> DiagonalGaussianDistribution: self._require_normalized_weights() - if not hasattr(self, "encoder"): + if self.encoder is None: raise RuntimeError("This MMAudio VAE was loaded decoder-only") if normalize_input: mel = self.normalize(mel) @@ -831,17 +343,7 @@ class MMAudioVAE(nn.Module): @torch.no_grad() def remove_weight_norm(self) -> MMAudioVAE: - # Idempotent: the diffusers reference calls this exactly once, in - # ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the - # freshly-constructed (random) Conv1d weights with a checkpoint's - # real, already-final values -- so a real checkpoint's own weights - # are never renormalized. Mirroring that timing here (see - # ``Kandinsky6AudioVAE.__init__``) means this method only ever runs - # against pre-load weights too; the guard below additionally makes a - # second, redundant call (e.g. the wrapper's own public - # ``remove_weight_norm()``) a true no-op instead of re-mutating an - # already-normalized (possibly checkpoint-loaded) weight a second - # time. + # normalize only random initialization; checkpoint weights are already final if self._weights_normalized: return self for name, module in self.named_modules(): diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py index af2d2359a2..f7db7368c3 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py @@ -38,15 +38,19 @@ from sglang.multimodal_gen.runtime.models.vaes.common import ( ) from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm -# Pixel frames per encode segment after the first (which has one more: the causal first frame). +# pixel frames per segment after the extra causal first frame _SEGMENT_FRAMES = 16 -# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles -# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9 -# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv -# workspace to a fraction of the activation. +# temporal chunks bound workspace and avoid int32 indexing overflow at 4x decode _MAX_CONV_NUMEL = 2 * 10**9 +def _segment_sizes(frames: int, stride: int) -> list[int]: + """The first causal segment has one extra frame; the final segment may be short.""" + return [min(frames, stride + 1)] + [ + min(stride, frames - start) for start in range(stride + 1, frames, stride) + ] + + class _SegmentCache: """Causal state carried from one temporal segment to the next during a single encode / decode call.""" @@ -56,17 +60,12 @@ class _SegmentCache: def _silu(x: torch.Tensor) -> torch.Tensor: - # The output norms use this form and the resblocks use F.silu; the two are not bitwise - # equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is - # deliberate, not a simplification opportunity. + # output norms require this form; F.silu in resblocks rounds differently in bf16 return x * torch.sigmoid(x) class _K6RMSNorm(WanRMS_norm): - """``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the - Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for - fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably - drifts from the reference over many resnet blocks.""" + """Wan RMS norm with the reference's unconditional fp32 reduction.""" def forward(self, x: torch.Tensor) -> torch.Tensor: normed = F.normalize(x.float(), dim=1).to(x.dtype) @@ -89,10 +88,7 @@ class _ChunkedConv3d(nn.Conv3d): f"padding {self.padding}" ) if self.stride[0] != 1: - # A strided conv can't reuse the stride-1 path's "prefix each chunk with the - # previous chunk's tail" trick below (a strided conv's output doesn't align with - # input chunk boundaries), so slide one kernel-width window at a time instead, only - # raising if even a single window would overflow. + # strided outputs may cross chunk boundaries: use one kernel window per output stride = self.stride[0] window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4] if window_numel >= _MAX_CONV_NUMEL: @@ -100,8 +96,6 @@ class _ChunkedConv3d(nn.Conv3d): f"frames are too big for Conv3d even one stride-{stride} window at a time " f"({window_numel} elements per window)" ) - # Bare super() doesn't resolve inside a list comprehension's own implicit scope, - # hence the loop. window_outputs = [] for i in range(0, frames - k + 1, stride): window_outputs.append(super().forward(x[:, :, i : i + k])) @@ -229,21 +223,14 @@ class _ResnetBlock3D(nn.Module): if in_channels != out_channels: self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1) - def _norm( - self, - norm: nn.Module, - h: torch.Tensor, - zq: torch.Tensor | None, - cache: _SegmentCache, - ) -> torch.Tensor: - return norm(h) if zq is None else norm(h, zq, cache) - def forward( self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None ) -> torch.Tensor: - h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True) + h = self.norm1(x) if zq is None else self.norm1(x, zq, cache) + h = F.silu(h, inplace=True) h = self.conv1(h, cache) - h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True) + h = self.norm2(h) if zq is None else self.norm2(h, zq, cache) + h = F.silu(h, inplace=True) h = self.conv2(h, cache) if hasattr(self, "nin_shortcut"): x = self.nin_shortcut(x) @@ -375,8 +362,6 @@ class _PXSUpsample(nn.Module): x = x[:, :, 1:] x = self.temporal_conv(x, cache) + x frames, height, width = x.shape[-3:] - # Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow - # this guards against (the old vendored code did not chunk this specific call). x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2)) x.add_(self.spatial_conv(x)) return self.linear(x) @@ -402,7 +387,6 @@ class _Encoder3D(nn.Module): temporal_compress_start_level: int, ) -> None: super().__init__() - self.num_res_blocks = num_res_blocks time_levels = range( temporal_compress_start_level, temporal_compress_start_level + int(math.log2(temporal_compress_times)), @@ -494,10 +478,7 @@ class _Decoder3D(nn.Module): return self.conv_out(_silu(self.norm_out(h, z, cache)), cache) -# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything -# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``, -# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored -# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params. +# constructor fields; resolution and training-only metadata do not affect the architecture _ARCH_KEYS = ( "ch", "ch_mult", @@ -508,12 +489,7 @@ _ARCH_KEYS = ( def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]: - # norm_type is the one legacy knob this port still checks: unlike padding_mode / - # downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer - # branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride / - # double-z behavior those knobs used to select), a group_norm checkpoint would silently - # get rms_norm layers instead of a clear error, which is the one mismatch worth failing - # loudly on here. + # never silently substitute RMSNorm for an unsupported group-norm checkpoint norm_type = conf.get("norm_type", "rms_norm") if norm_type != "rms_norm": raise ValueError( @@ -596,12 +572,7 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin): def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]: """``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``.""" - split_list = [_SEGMENT_FRAMES + 1] - remaining = x.size(2) - split_list[0] - while remaining > 0: - split_list.append(_SEGMENT_FRAMES) - remaining -= _SEGMENT_FRAMES - split_list[-1] += remaining + split_list = _segment_sizes(x.size(2), _SEGMENT_FRAMES) cache = _SegmentCache() latents = [] for segment in torch.split(x, split_list, dim=2): @@ -634,15 +605,9 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin): return DecoderOutput(sample=self._decode(z)) def _decode(self, z: torch.Tensor) -> torch.Tensor: - segment = _SEGMENT_FRAMES // self.temporal_compression - num_frames = z.size(2) - if num_frames == 1: - split_list = [1] - else: - split_list = [segment] * ((num_frames - 1) // segment) - if (num_frames - 1) % segment: - split_list.append((num_frames - 1) % segment) - split_list[0] += 1 + split_list = _segment_sizes( + z.size(2), _SEGMENT_FRAMES // self.temporal_compression + ) cache = _SegmentCache() samples = [] for chunk in torch.split(z, split_list, dim=2): diff --git a/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py index 57148dee39..4af304027c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py @@ -15,22 +15,12 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( ComposedPipelineBase, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import ( Kandinsky6SRDecodeStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( Kandinsky6SRDenoisingStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import ( Kandinsky6SREncodeStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( Kandinsky6SRInputStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( Kandinsky6SRLatentPrepStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import ( Kandinsky6SROutputStage, ) from sglang.multimodal_gen.runtime.server_args import ServerArgs diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py index ae61db06cf..d0190ce960 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py @@ -29,6 +29,9 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import ( @@ -48,17 +51,8 @@ from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYP class Kandinsky6DenoisingStage(DenoisingStage): """Run the Kandinsky6 joint video+audio denoising loop.""" - def __init__(self, transformer, scheduler, pipeline=None) -> None: - super().__init__( - transformer=transformer, scheduler=scheduler, pipeline=pipeline - ) - def _owns_compile_warmup_lifecycle(self) -> bool: - # forward() overrides DenoisingStage.forward (the joint video+audio - # loop cannot reuse the parent's single-tensor _denoise()), but it - # still wraps the loop in `_offload_for_torch_compile_warmup` itself - # below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's - # other joint-modality denoising stage. + # forward wraps the joint loop in the shared compile warmup lifecycle return True def component_uses( @@ -155,7 +149,6 @@ class Kandinsky6DenoisingStage(DenoisingStage): t_expand: torch.Tensor, visual_rope_pos: list[torch.Tensor], scale_factor: tuple[float, ...], - sparse_params: dict[str, Any] | None, visual_token_type_ids: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor]: """Run sequential or distributed CFG branches and combine video/audio velocity.""" @@ -172,7 +165,6 @@ class Kandinsky6DenoisingStage(DenoisingStage): timestep=t_expand, visual_rope_pos=visual_rope_pos, scale_factor=scale_factor, - sparse_params=sparse_params, visual_token_type_ids=visual_token_type_ids, # BCG clones tuple leaves so the next CFG replay cannot # overwrite the previous branch's predictions @@ -227,7 +219,7 @@ class Kandinsky6DenoisingStage(DenoisingStage): scheduler = batch.scheduler if scheduler is None: raise ValueError("scheduler must be set for Kandinsky6 denoising.") - use_piflow = bool(getattr(scheduler, "is_piflow", False)) + use_piflow = isinstance(scheduler, PiflowScheduler) if use_piflow and ( not math.isfinite(batch.guidance_scale) or abs(batch.guidance_scale - 1.0) > 1e-6 @@ -237,30 +229,10 @@ class Kandinsky6DenoisingStage(DenoisingStage): pipeline_config = server_args.pipeline_config arch = pipeline_config.dit_config.arch_config - # The runtime transformer already rejects attention_engine="nabla" - # at construction time (NABLA sparse attention is not yet ported), - # so sparse_params stays None on every currently-reachable - # configuration. A future NABLA-capable DiT would fill this in. - if arch.attention_engine == "nabla": - raise NotImplementedError( - "Kandinsky6 NABLA sparse-attention metadata construction is not implemented; " - "the runtime transformer also rejects attention_engine='nabla' at construction " - "time. Use attention_engine='auto' or 'sdpa'." - ) - sparse_params = None - device = get_local_torch_device() target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision] - # .clone(): batch.latents was created by Kandinsky6LatentPreparationStage - # under an active torch.inference_mode() context further up the - # pipeline, which marks it as an "inference tensor" -- such tensors - # can only be mutated in place while still inside an inference_mode - # context. This stage runs under plain @torch.no_grad() instead, and - # mutates `video` in place every denoising step (video[..., :num_channels] - # = ...), so clone once here to get an ordinary, freely-mutable tensor - # before the loop starts. audio is never mutated in place (only - # reassigned via out-of-place `+`), so it needs no clone. + # clone inference tensors before the in-place video updates under no_grad video = batch.latents.clone() audio = batch.audio_latents num_channels = int(arch.in_visual_dim) @@ -295,40 +267,25 @@ class Kandinsky6DenoisingStage(DenoisingStage): spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio patch_size = arch.patch_size - # Constant per-request geometry -- computed once outside the loop, - # not per-step. + # geometry is constant across steps num_video_frames = video.shape[1] - (1 if tail_cond else 0) t_positions = torch.arange(num_video_frames, device=device) if tail_cond: - # The appended reference frame reuses T-position 0's rope row - # (RoPE3D is a pure position -> table lookup, so duplicating the - # first row is equivalent to the diffusers reference's - # ``torch.cat([rope, rope[:1]])``). + # the reference frame reuses time position zero t_positions = torch.cat([t_positions, t_positions.new_zeros(1)]) visual_rope_pos = [ t_positions, torch.arange(height // spatial_ratio // patch_size[1], device=device), torch.arange(width // spatial_ratio // patch_size[2], device=device), ] - # Fixed per-checkpoint RoPE frequency scaling read from the DiT's - # arch config (transformer/config.json's "scale_factor", both real - # Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers - # reference, which resolves this once in - # ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every - # request regardless of the request's own height/width. NOT a - # function of the request's resolution. + # RoPE scaling comes from the checkpoint, not the request resolution scale_factor = arch.scale_factor image_latent = batch.image_latent total_steps = int(batch.timesteps.shape[0]) self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch) - # PiFlow's noisy video state is tracked in fp32 across steps here, - # separately from the `video` buffer's bf16 storage dtype -- see the - # module docstring. Always a Tensor (not `Tensor | None`) even on the - # flow-Euler path, where it is simply unused: the initial cast is one - # cheap op, and keeping it non-Optional avoids re-narrowing it after - # every `if use_piflow:` branch re-entry in the loop below. + # pi-Flow retains fp32 state between bf16 DiT calls video_state = video[..., :num_channels].to(torch.float32) with self.use_declared_component( @@ -371,7 +328,6 @@ class Kandinsky6DenoisingStage(DenoisingStage): t_expand=t_expand, visual_rope_pos=visual_rope_pos, scale_factor=scale_factor, - sparse_params=sparse_params, visual_token_type_ids=visual_token_type_ids, ) @@ -397,9 +353,7 @@ class Kandinsky6DenoisingStage(DenoisingStage): if use_piflow: video_state[:, -1:] = ref_frame.to(torch.float32) - # Manual Euler update for audio, using the SAME per-step - # sigma delta the scheduler.step() call above just - # consumed for video. + # Euler audio reuses the video's sigma delta without advancing it if use_piflow: audio = audio_scheduler.step( audio_vel, timestep, audio, return_dict=False diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py index f21295aef7..6990589aa0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py @@ -1,8 +1,2 @@ # SPDX-License-Identifier: Apache-2.0 -"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers. - -Import the stage classes from their modules (``input_stage``, ``encode_stage``, -``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package -``__init__`` stays empty so that configs can import the pure helpers (``video_utils``, -``tiling``, ...) cheaply. -""" +"""SR stages and helpers; import stages explicitly to keep config imports lightweight.""" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py deleted file mode 100644 index 0a0237a42a..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py +++ /dev/null @@ -1,78 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles.""" - -import torch - -from sglang.multimodal_gen.runtime.distributed import ( - get_decode_parallel_group_coordinator, - get_local_torch_device, -) -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - ComponentUse, -) -from sglang.multimodal_gen.runtime.models.vaes.common import ( - can_install_spatial_shard_parallel_decode, - has_decode_parallel_world, -) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_DENOISED_KEY, - SR_TILES_KEY, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - decode_chunks, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.utils.precision import resolve_precision - - -class Kandinsky6SRDecodeStage(PipelineStage): - """VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``).""" - - def __init__(self, vae) -> None: - super().__init__() - self.vae = vae - - def component_uses( - self, server_args: ServerArgs, stage_name: str | None = None - ) -> list[ComponentUse]: - vae_dtype = resolve_precision( - server_args, "vae", precision_attr="vae_precision" - ) - return [ - ComponentUse( - self._component_stage_name(stage_name), - "vae", - phase="decode_tiles", - target_dtype=vae_dtype, - ) - ] - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - denoised = batch.extra.pop(SR_DENOISED_KEY) - device = get_local_torch_device() - group = None - if ( - server_args.pipeline_config.vae_config.use_parallel_tiling - and has_decode_parallel_world() - and not can_install_spatial_shard_parallel_decode( - server_args.pipeline_config.vae_config - ) - ): - group = get_decode_parallel_group_coordinator() - with self.use_declared_component( - component_name="vae", module=self.vae, phase="decode_tiles" - ) as vae: - assert vae is not None - self.vae = vae - tiles = decode_chunks( - denoised, - vae, - scaling_factor=vae.scaling_factor, - device=device, - group=group, - ) - batch.extra[SR_TILES_KEY] = tiles - return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py deleted file mode 100644 index ce48fb2e8c..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks. - -Each chunk resets its scheduler; the generic single-latent CFG loop and -prompt-padding BCG path do not apply.""" - -from __future__ import annotations - -import torch - -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device -from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - ComponentUse, -) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_CHUNKS_KEY, - SR_DENOISED_KEY, - SR_DIT_SPEC_KEY, - SR_SAMPLING_SPEC_KEY, - effective_scheduler, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - denoise_chunks, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( - StageValidators as V, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( - VerificationResult, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype - - -class Kandinsky6SRDenoisingStage(DenoisingStage): - """Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage.""" - - def __init__(self, transformer, scheduler, pipeline=None) -> None: - super().__init__( - transformer=transformer, - scheduler=scheduler, - pipeline=pipeline, - transformer_2=None, - vae=None, - ) - - def _owns_compile_warmup_lifecycle(self) -> bool: - # ``forward`` below does not go through the shared ``_denoise`` loop that this guard - # normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage): - # the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``. - return True - - def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: - """Validate tile-chunk state instead of the shared text/CFG loop inputs.""" - result = VerificationResult() - result.add_check( - "extra[SR_SAMPLING_SPEC_KEY]", - batch.extra.get(SR_SAMPLING_SPEC_KEY), - V.not_none, - ) - result.add_check( - "extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none - ) - result.add_check( - "extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty - ) - return result - - def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: - result = VerificationResult() - result.add_check( - "extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty - ) - return result - - def component_uses( - self, server_args: ServerArgs, stage_name: str | None = None - ) -> list[ComponentUse]: - dit_dtype = precision_to_dtype( - server_args.pipeline_config.dit_precision, "dit_precision" - ) - return [ - ComponentUse( - self._component_stage_name(stage_name), - "transformer", - phase="denoise_tiles", - target_dtype=dit_dtype, - preferred_ready_after_request=True, - memory_intensive=True, - ) - ] - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - with self._offload_for_torch_compile_warmup(batch): - return self._denoise_tiles(batch, server_args) - - def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req: - spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY) - dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY) - chunks = batch.extra.pop(SR_CHUNKS_KEY) - device = get_local_torch_device() - - # Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the - # corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this - # request's DiT-call count, the closest analogue of the global ``num_inference_steps`` - # those hooks expect. - self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch) - - scheduler = effective_scheduler(spec, self.scheduler) - with self.use_declared_component( - component_name="transformer", module=self.transformer, phase="denoise_tiles" - ) as transformer: - assert transformer is not None - self.transformer = transformer - total = spec.steps_per_chunk * len(chunks) - with self.progress_bar( - total=total, batch=batch, desc="Kandinsky6 SR denoising" - ) as progress: - denoised = denoise_chunks( - chunks, - transformer, - scheduler, - dit_spec=dit_spec, - spec=spec, - device=device, - step_context=lambda step: set_forward_context( - current_timestep=step, attn_metadata=None, forward_batch=batch - ), - on_step=progress.update, - ) - self._finish_active_component_use() - batch.extra[SR_DENOISED_KEY] = denoised - return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py deleted file mode 100644 index 97c554f1ef..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR.""" - -import torch - -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - ComponentUse, -) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_LR_LATENT_KEY, - SR_TILING_SCALE_KEY, - SR_VIDEO_KEY, - uses_latent_path, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - encode_video_to_lr_latent, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.utils.precision import resolve_precision - - -class Kandinsky6SREncodeStage(PipelineStage): - """Encode the whole clip only for the LU path; pixel mode encodes tiles later.""" - - def __init__(self, vae, latent_upscaler=None) -> None: - super().__init__() - self.vae = vae - # Only consulted to pick the path (does the bank have this scale?). - self.latent_upscaler = latent_upscaler - - def component_uses( - self, server_args: ServerArgs, stage_name: str | None = None - ) -> list[ComponentUse]: - vae_dtype = resolve_precision( - server_args, "vae", precision_attr="vae_precision" - ) - return [ - ComponentUse( - self._component_stage_name(stage_name), - "vae", - phase="encode_video", - target_dtype=vae_dtype, - start_at_stage_entry=False, - ) - ] - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - tiling_scale = batch.extra[SR_TILING_SCALE_KEY] - if not uses_latent_path(self.latent_upscaler, tiling_scale): - return batch - video = batch.extra.pop(SR_VIDEO_KEY) - with self.use_declared_component( - component_name="vae", module=self.vae, phase="encode_video" - ) as vae: - assert vae is not None - self.vae = vae - lr_latent = encode_video_to_lr_latent( - video, vae, device=get_local_torch_device() - ) - batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu() - return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py deleted file mode 100644 index 6bb2936b9f..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio).""" - -import os - -import torch - -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_REQUESTED_HW_KEY, - SR_TILING_SCALE_KEY, - SR_VIDEO_KEY, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( - VAE_SPATIAL_FACTOR, - pad_to_spatial_factor, - pre_upscale_video, - resolve_scale_request, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( - SOURCE_AUDIO_SAMPLE_RATE, - DecodedClip, - decode_clip, - extract_source_audio, - synthetic_clip, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs - -# Warmup requests carry no (usable) video: run the real path on a small synthetic clip. -WARMUP_CLIP_FRAMES = 9 -WARMUP_CLIP_HW = (256, 384) - - -class Kandinsky6SRInputStage(PipelineStage): - """Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio. - - Record requested dimensions before VAE padding so output can remove it.""" - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale) - clip = self._load_clip(batch) - video = clip.frames - if pre_upscale != 1.0: - video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR) - frames = video.shape[0] - # Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel - # pre-upscale happens to already land on an aligned size): without this, a source - # whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4, - # which runs no pixel resize at all) fails partway through the whole-video KVAE - # encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps - # every source pixel; batch.height / batch.width below, and the output stage's final - # crop, restore the originally requested dimensions. - video, (requested_h, requested_w) = pad_to_spatial_factor( - video, VAE_SPATIAL_FACTOR - ) - - batch.extra[SR_VIDEO_KEY] = video - batch.extra[SR_TILING_SCALE_KEY] = tiling_scale - batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w) - batch.fps = clip.fps - batch.num_frames = frames - batch.height = requested_h * tiling_scale - batch.width = requested_w * tiling_scale - self._attach_source_audio(batch, frames=frames, fps=clip.fps) - return batch - - def _load_clip(self, batch: Req) -> DecodedClip: - if batch.is_warmup: - height, width = WARMUP_CLIP_HW - frames = synthetic_clip( - height=height, width=width, frames=WARMUP_CLIP_FRAMES - ) - return DecodedClip(frames=frames, fps=24, source_fps=24.0) - path = batch.sampling_params.source_video_path() - if path is None or not os.path.isfile(path): - raise ValueError(f"Kandinsky6 SR input video not found: {path!r}") - self.log_info("Decoding %s", path) - return decode_clip(path) - - @staticmethod - def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None: - if batch.is_warmup: - return - path = batch.sampling_params.source_video_path() - audio = extract_source_audio( - path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps - ) - if audio is None: - batch.audio = None - batch.audio_sample_rate = None - return - batch.audio = torch.from_numpy(audio) - batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py deleted file mode 100644 index c87180e817..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback. - -Validate sampler/head compatibility and step count before processing tiles.""" - -from functools import partial - -import torch - -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - ComponentUse, -) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_CHUNKS_KEY, - SR_DIT_SPEC_KEY, - SR_LR_LATENT_KEY, - SR_PLAN_KEY, - SR_SAMPLING_SPEC_KEY, - SR_TILING_SCALE_KEY, - SR_VIDEO_KEY, - build_dit_spec, - build_sampling_spec, - check_denoising_request, - uses_latent_path, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( - DitSpec, - SamplingSpec, - module_dtype, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - TilePlan, - plan_tiles, - prepare_lu_tile_latents, - prepare_pixel_tile_latents, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( - VAE_SPATIAL_FACTOR, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.utils.precision import ( - resolve_precision, -) - - -class Kandinsky6SRLatentPrepStage(PipelineStage): - """LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time.""" - - def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None: - super().__init__() - self.vae = vae - self.transformer = transformer # arch-config / dtype only: never called here - self.latent_upscaler = latent_upscaler - self.scheduler = ( - scheduler # resolved sampler validation only: never stepped here - ) - - def component_uses( - self, server_args: ServerArgs, stage_name: str | None = None - ) -> list[ComponentUse]: - stage_name = self._component_stage_name(stage_name) - vae_dtype = resolve_precision( - server_args, "vae", precision_attr="vae_precision" - ) - uses: list[ComponentUse] = [] - if self.latent_upscaler is not None: - uses.append( - ComponentUse( - stage_name, - "latent_upscaler", - phase="upscale_tiles", - target_dtype=torch.bfloat16, - ) - ) - uses.append( - ComponentUse( - stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype - ) - ) - return uses - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> Req: - arch = server_args.pipeline_config.dit_config.arch_config - tiling_scale = batch.extra[SR_TILING_SCALE_KEY] - seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0] - spec = build_sampling_spec( - arch=arch, - tiling_scale=tiling_scale, - seed=seed, - num_steps=batch.num_inference_steps, - tiles_batch_size=batch.sr_tiles_batch_size, - tile_min_overlap=batch.sr_tile_min_overlap, - scheduler=self.scheduler, - ) - check_denoising_request(spec, batch.num_inference_steps) - dit_spec = build_dit_spec(self.transformer) - device = get_local_torch_device() - - plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device) - batch.extra[SR_PLAN_KEY] = plan - batch.extra[SR_CHUNKS_KEY] = chunks - batch.extra[SR_SAMPLING_SPEC_KEY] = spec - batch.extra[SR_DIT_SPEC_KEY] = dit_spec - return batch - - def _prepare_chunks( - self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device - ) -> tuple[TilePlan, list[torch.Tensor]]: - common = dict( - visual_size=spec.visual_size, - tiling_scale=spec.tiling_scale, - tile_min_overlap=spec.tile_min_overlap, - ) - if uses_latent_path(self.latent_upscaler, spec.tiling_scale): - lr_latent = batch.extra.pop(SR_LR_LATENT_KEY) - plan = plan_tiles( - frame_hw=( - lr_latent.shape[-2] * VAE_SPATIAL_FACTOR, - lr_latent.shape[-1] * VAE_SPATIAL_FACTOR, - ), - **common, - ) - with self.use_declared_component( - component_name="latent_upscaler", - module=self.latent_upscaler, - phase="upscale_tiles", - ) as bank: - assert bank is not None - self.latent_upscaler = bank - chunks = prepare_lu_tile_latents( - lr_latent, - plan, - upscale_fn=partial(bank.upscale, scale=spec.tiling_scale), - lu_dtype=module_dtype(bank), - scaling_factor=self.vae.scaling_factor, - dit_spec=dit_spec, - spec=spec, - device=device, - ) - return plan, chunks - - video = batch.extra.pop(SR_VIDEO_KEY) - plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common) - with self.use_declared_component( - component_name="vae", module=self.vae, phase="encode_tiles" - ) as vae: - assert vae is not None - self.vae = vae - chunks = prepare_pixel_tile_latents( - video, - plan, - vae=vae, - scaling_factor=vae.scaling_factor, - dit_spec=dit_spec, - spec=spec, - device=device, - ) - return plan, chunks diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py index 297fcdb4d6..7c3057e7fd 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py @@ -35,23 +35,12 @@ def degrade_lq_latent( return (1 - noise_scale) * lq_latent + noise_scale * eps -def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor: - """Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype).""" - if dtype is not None and value.is_floating_point() and value.dtype != dtype: - return value.to(dtype=dtype) - return value - - -def build_initial_latent( # noqa: PLR0913 +def build_initial_latent( *, instruct_type: InstructType, visual_cond: bool, in_visual_dim: int, lq_latent: torch.Tensor, - batch_size: int, - duration: int, - height: int, - width: int, device: str | int | torch.device, seed: int, lq_noise_scale: float, @@ -59,76 +48,33 @@ def build_initial_latent( # noqa: PLR0913 lq_channel_noise_scale: float, dtype: torch.dtype | None = None, ) -> torch.Tensor: - """Build [starting | conditioning | mask] for the selected instruction mode. - - Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except - unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise.""" - lq_latent = lq_latent.to(device) - lq_latent = _cast_floating(lq_latent, dtype) - + """Build packed [B*T, H, W, C] noise and optional conditioning without reseeding global RNG.""" + if instruct_type not in ("noise", "channel", "hybrid", "hybrid_anchor"): + raise ValueError(f"Unsupported SR instruct_type={instruct_type!r}") + lq_latent = lq_latent.to(device=device, dtype=dtype) + generator = torch.Generator(device=device).manual_seed(seed) + if instruct_type == "channel": + starting = torch.randn( + (*lq_latent.shape[:-1], in_visual_dim), device=device, generator=generator + ) + else: + starting = degrade_lq_latent( + lq_latent, lq_noise_scale, lq_noise_type, generator=generator + ) if instruct_type == "noise": - g_noise = torch.Generator(device=device) - g_noise.manual_seed(seed) - degraded_lq = degrade_lq_latent( - lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise - ) if not visual_cond: - # No conditioning channels in the model input layer (e.g. the - # t2v-initialized KVAE SR DiT) — feed the bare C-channel latent. - return degraded_lq - zero_cond = torch.zeros_like(degraded_lq) - zero_mask = torch.zeros( - [*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device + return starting + conditioning = torch.zeros_like(starting) + mask = torch.zeros_like(starting[..., :1]) + elif instruct_type == "hybrid_anchor": + # no HR anchor exists at inference; conditioning noise still uses the global RNG + conditioning = degrade_lq_latent( + torch.zeros_like(lq_latent), lq_channel_noise_scale, noise_type="linear" ) - return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1) - - if instruct_type in ("channel", "hybrid"): - if instruct_type == "hybrid": - g_noise = torch.Generator(device=device) - g_noise.manual_seed(seed) - starting = degrade_lq_latent( - lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise - ) - else: - g = torch.Generator(device=device) - g.manual_seed(seed) - starting = torch.randn( - batch_size * duration, - height, - width, - in_visual_dim, - device=device, - generator=g, - ) - channel_lq = degrade_lq_latent( + mask = torch.zeros_like(lq_latent[..., :1]) + else: + conditioning = degrade_lq_latent( lq_latent, lq_channel_noise_scale, noise_type="linear" ) mask = torch.ones_like(lq_latent[..., :1]) - return torch.cat([starting, channel_lq, mask], dim=-1) - - if instruct_type == "hybrid_anchor": - # Anchor-free inference (tiled real-world LQ has no HR ground truth): - # a zeroed anchor + zeroed mask runs the model with no anchor signal, - # equivalent to anchor_indices=[] for the packed-latent datasets. - anchor_latent = torch.zeros_like(lq_latent) - anchor_mask = torch.zeros_like(lq_latent[..., :1]) - g_noise = torch.Generator(device=device) - g_noise.manual_seed(seed) - starting = degrade_lq_latent( - lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise - ) - anchor = degrade_lq_latent( - anchor_latent, lq_channel_noise_scale, noise_type="linear" - ) - return torch.cat([starting, anchor, anchor_mask], dim=-1) - - msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}" - raise ValueError(msg) - - -__all__ = [ - "InstructType", - "LqNoiseType", - "build_initial_latent", - "degrade_lq_latent", -] + return torch.cat([starting, conditioning, mask], dim=-1) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py deleted file mode 100644 index 6b1e8e5180..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Stitch / crop / resize / output stage of Kandinsky 6 video SR.""" - -import torch - -from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( - resolve_target_hw, -) -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_PLAN_KEY, - SR_REQUESTED_HW_KEY, - SR_TILES_KEY, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - stitch_tiles, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( - crop_to_hw, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( - to_output_video, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( - resize_to_target, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs - - -class Kandinsky6SROutputStage(PipelineStage): - """Hann-blends the decoded tiles, restores the requested output size, applies the - optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1]) - plus the mono source audio. - - CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set - to the final size, ``batch.fps`` was set by the input stage (and now also carried on - the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save - path unchanged). - """ - - @torch.no_grad() - def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: - tiles = batch.extra.pop(SR_TILES_KEY) - plan = batch.extra.pop(SR_PLAN_KEY) - video = stitch_tiles(tiles, plan) - del tiles - - requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None) - if requested_hw is not None: - requested_h, requested_w = requested_hw - video = crop_to_hw( - video, - (requested_h * plan.tiling_scale, requested_w * plan.tiling_scale), - ) - - target_hw = resolve_target_hw( - batch.sr_target_resolution, - tuple(video.shape[-2:]), - batch.sr_target_resize_mode, - ) - if target_hw is not None: - video = resize_to_target(video, target_hw) - batch.height, batch.width = video.shape[-2:] - return OutputBatch( - output=to_output_video(video), - audio=batch.audio, - audio_sample_rate=batch.audio_sample_rate, - fps=batch.fps, - metrics=batch.metrics, - usage=batch.usage, - ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py index 13711d5430..a9604f8628 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py @@ -16,7 +16,6 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( DitSpec, - PiflowParams, SamplingSpec, module_dtype, ) @@ -31,10 +30,7 @@ SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4) SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path) SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb] -# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out -# of the single combined stage so the denoising stage can subclass the shared DenoisingStage -# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain -# data, like TilePlan above), and the chunk lists are CPU tensors throughout. +# stages exchange immutable specs and CPU chunks, not resident model objects SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec SR_CHUNKS_KEY = ( @@ -43,11 +39,7 @@ SR_CHUNKS_KEY = ( SR_DENOISED_KEY = ( "kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise) ) -# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the -# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale -# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The -# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to -# (H * scale, W * scale) so the delivered video matches what was actually asked for. +# original (H, W), used to crop VAE alignment padding from the stitched output SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw" @@ -71,7 +63,56 @@ def build_dit_spec(dit: Any) -> DitSpec: ) -def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]: +def build_sampling_spec( + *, + arch: Kandinsky6SRArchConfig, + tiling_scale: int, + seed: int, + num_steps: int, + tiles_batch_size: int, + tile_min_overlap: float, + scheduler: FlowMatchEulerDiscreteScheduler, +) -> SamplingSpec: + """Validate the loaded scheduler and resolve DiT calls before encoding tiles.""" + if num_steps < 1: + raise ValueError( + f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_steps}" + ) + if not isinstance(scheduler, FlowMatchEulerDiscreteScheduler): + raise ValueError( + "Kandinsky6 SR requires a FlowMatchEulerDiscreteScheduler or " + "PiflowScheduler from the checkpoint" + ) + is_piflow = isinstance(scheduler, PiflowScheduler) + n_grid = scheduler.config.n_grid if is_piflow else 1 + expected_width = arch.in_visual_dim * n_grid + if arch.out_visual_dim != expected_width: + raise ValueError( + f"Kandinsky6 SR: {type(scheduler).__name__} requires a {expected_width}-channel " + f"head, but out_visual_dim={arch.out_visual_dim}; use the scheduler " + "from the same checkpoint as the transformer" + ) + if is_piflow: + nfe = scheduler.config.nfe + if nfe is None or nfe < 1: + raise ValueError( + f"Kandinsky6 SR: PiflowScheduler requires nfe >= 1, got {nfe}" + ) + if arch.sr_cap_noise_timestep and arch.effective_override("instruct_type") in ( + "noise", + "hybrid", + ): + raise NotImplementedError( + "piflow sampler does not support cap_noise_timestep for noise/hybrid instruct" + ) + if num_steps != nfe: + logger.warning( + "Kandinsky6 SR: ignoring num_inference_steps=%d; the pi-Flow checkpoint " + "requires %d DiT calls per tile", + num_steps, + nfe, + ) + num_steps = nfe table = arch.sr_scale_factor visual_size = arch.sr_visual_size[0] if table is None: @@ -84,138 +125,17 @@ def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, floa raise ValueError( f"sr_scale_factor has no entry for visual size {visual_size}: {table}" ) - return tuple(float(value) for value in values) - - -def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None: - """pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler).""" - if not isinstance(scheduler, PiflowScheduler): - return None - config = scheduler.config - if config["nfe"] is None: - return None - return PiflowParams( - nfe=int(config["nfe"]), - num_policy_substeps=int(config["num_policy_substeps"]), - final_step_size_scale=float(config["final_step_size_scale"]), - shift=float(config["shift"]), - n_grid=int(config["n_grid"]), - eps=float(config["eps"]), - ) - - -def _describe_scheduler(scheduler: Any) -> str: - if scheduler is None: - return "missing" - if isinstance(scheduler, PiflowScheduler): - return "a PiflowScheduler without nfe" - return f"a {type(scheduler).__name__}" - - -def _check_sampler_fits_head( - *, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any -) -> None: - """Reject incompatible scheduler and DiT head widths before encoding tiles.""" - head, grid = arch.head_width, arch.base_out_visual_dim - if piflow is None: - if head != grid: - raise ValueError( - f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / " - f"pi-Flow head: {grid} channels per latent), but the sampler resolved " - "to flow-Euler: the scheduler component is " - f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with " - "nfe selects pi-Flow. Use the scheduler of the same repo as the " - "transformer." - ) - return - if piflow.nfe < 1: - raise ValueError( - "Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, " - f"got nfe={piflow.nfe}." - ) - if head != piflow.n_grid * grid: - raise ValueError( - f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid " - f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = " - f"{piflow.n_grid * grid} channel DX head, but the transformer head is " - f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The " - "scheduler and the transformer come from different repos." - ) - - -def build_sampling_spec( - *, - arch: Kandinsky6SRArchConfig, - tiling_scale: int, - seed: int, - num_steps: int, - tiles_batch_size: int, - tile_min_overlap: float, - scheduler: Any = None, -) -> SamplingSpec: - """Resolve sampling from the bundle scheduler, falling back to legacy DiT fields. - - num_steps counts DiT calls, not Diffusers timestep grid points.""" - piflow = piflow_params_from_scheduler(scheduler) - if piflow is None and arch.is_piflow: - piflow = PiflowParams( - nfe=arch.piflow_nfe, - num_policy_substeps=arch.piflow_num_policy_substeps, - final_step_size_scale=arch.piflow_final_step_size_scale, - shift=arch.piflow_shift, - n_grid=arch.n_grid, - eps=arch.piflow_eps, - ) - _check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler) return SamplingSpec( tiling_scale=tiling_scale, tiles_batch_size=tiles_batch_size, seed=seed, num_steps=num_steps, + is_piflow=is_piflow, tile_min_overlap=tile_min_overlap, - visual_size=arch.sr_visual_size[0], - scale_factor=_rope_scale_factor(arch), - scheduler_scale=arch.sr_scheduler_scale, + visual_size=visual_size, + scale_factor=tuple(float(value) for value in values), lq_noise_scale=arch.sr_lq_noise_scale, lq_noise_type=arch.sr_lq_noise_type, lq_channel_noise_scale=arch.sr_lq_channel_noise_scale, cap_noise_timestep=arch.sr_cap_noise_timestep, - piflow=piflow, ) - - -def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any: - """Use the loaded scheduler when compatible; synthesize one for legacy configs.""" - if spec.piflow is not None: - if isinstance(scheduler, PiflowScheduler): - return scheduler - return PiflowScheduler( - nfe=spec.piflow.nfe, - n_grid=spec.piflow.n_grid, - shift=spec.piflow.shift, - eps=spec.piflow.eps, - final_step_size_scale=spec.piflow.final_step_size_scale, - num_policy_substeps=spec.piflow.num_policy_substeps, - ) - if isinstance(scheduler, FlowMatchEulerDiscreteScheduler): - return scheduler - return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale) - - -def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None: - """Validate the requested DiT-call count before denoising. - - Diffusers counts timestep grid points instead: its default 5 is 4 calls here.""" - if num_inference_steps < 1: - raise ValueError( - f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}" - ) - if spec.piflow is not None and num_inference_steps != spec.piflow.nfe: - logger.warning( - "Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow " - "bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow " - "bundles and every tile will run exactly nfe=%d DiT calls instead.", - num_inference_steps, - spec.piflow.nfe, - spec.piflow.nfe, - ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py index b6d7287558..24869d8e11 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py @@ -1,191 +1,133 @@ # SPDX-License-Identifier: Apache-2.0 """Scheduler-driven denoising for Kandinsky 6 SR tile chunks.""" -from collections.abc import Callable +from collections.abc import Callable, Sequence from contextlib import AbstractContextManager, nullcontext from typing import Any import msgspec import torch -# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output -DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor] -StepCallback = Callable[[], None] -StepContext = Callable[[int], AbstractContextManager] - - -class PiflowParams(msgspec.Struct, frozen=True, kw_only=True): - """pi-Flow parameters from the scheduler or legacy flat DiT config.""" - - nfe: int - num_policy_substeps: int - final_step_size_scale: float - shift: float - n_grid: int - eps: float - class DitSpec(msgspec.Struct, frozen=True, kw_only=True): - """Post-override behaviour of the DiT that the sampling code needs.""" + """Post-override input layout and loaded parameter dtype.""" instruct_type: str | None visual_cond: bool in_visual_dim: int use_motion_score: bool patch_size: tuple[int, int, int] - # The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the - # LQ latent to this dtype before building the initial latent, so a bf16-loaded - # checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT, - # e.g. a mocked one in a test) keeps the LQ latent's own dtype. dtype: torch.dtype | None = None class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True): - """Everything besides the components that defines one tiled SR run.""" + """Serializable settings shared by tile preparation and denoising.""" tiling_scale: int tiles_batch_size: int seed: int num_steps: int + is_piflow: bool tile_min_overlap: float visual_size: int scale_factor: tuple[float, float, float] - scheduler_scale: float lq_noise_scale: float lq_noise_type: str lq_channel_noise_scale: float cap_noise_timestep: bool - piflow: PiflowParams | None - - @property - def steps_per_chunk(self) -> int: - """DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``.""" - return self.piflow.nfe if self.piflow is not None else self.num_steps def bf16_autocast(device: torch.device | str) -> AbstractContextManager: - """CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models.""" if torch.device(device).type != "cuda": return nullcontext() return torch.autocast(device_type="cuda", dtype=torch.bfloat16) def module_dtype(module: torch.nn.Module) -> torch.dtype | None: - """Dtype of the first parameter (``None`` for a parameter-less module).""" return next((param.dtype for param in module.parameters()), None) def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor: - """Cast a floating tensor to the module's parameter dtype (reference helper).""" dtype = module_dtype(module) - if dtype is not None and value.is_floating_point() and value.dtype != dtype: - return value.to(dtype=dtype) - return value - - -def euler_start_timestep( - *, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None -) -> float: - """Euler start time: the LQ noise level when capped for noise-type conditioning.""" - capped = cap_noise_timestep and instruct_type in ( - "noise", - "hybrid", - "hybrid_anchor", + return ( + value.to(dtype=dtype) + if dtype is not None and value.is_floating_point() + else value ) - return lq_noise_scale if capped else 1.0 - - -def check_sampler_options( - *, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None -) -> None: - """Reject option combinations the reference sampler rejects.""" - if ( - piflow is not None - and cap_noise_timestep - and instruct_type in ("noise", "hybrid") - ): - raise NotImplementedError( - "piflow sampler does not support cap_noise_timestep for noise/hybrid " - "instruct" - ) - - -def make_dit_fn( - dit: Callable[..., torch.Tensor], - *, - latent_frames_hw: tuple[int, int, int], - patch_size: tuple[int, int, int], - scale_factor: tuple[float, float, float], - use_motion_score: bool, - step_context: StepContext | None = None, -) -> DitFn: - """Bind per-chunk RoPE/motion inputs and an optional per-step context.""" - frames, height, width = latent_frames_hw - context = step_context or (lambda step: nullcontext()) - - def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor: - rope_pos = [ - torch.arange(frames, device=x.device), - torch.arange(height // patch_size[1], device=x.device), - torch.arange(width // patch_size[2], device=x.device), - ] - motion = ( - torch.full((1,), 900.0, device=x.device, dtype=x.dtype) - if use_motion_score - else None - ) - with context(step): - return dit( - x, - model_time, - rope_pos, - scale_factor=scale_factor, - motion_score=motion, - ) - - return call @torch.no_grad() -def denoise_with_scheduler( - x: torch.Tensor, - dit_fn: DitFn, - scheduler: Any, - *, - channels: int, - is_piflow: bool, - on_step: StepCallback | None = None, -) -> torch.Tensor: - """Denoise one chunk after set_timesteps; keep conditioning channels fixed. - - Promote predictions to fp32 so flow-Euler does not downcast its running - state to the bf16 prediction dtype. pi-Flow already preserves the state dtype.""" - state = x[..., :channels].float() - cond = x[..., channels:] - for step, t in enumerate(scheduler.timesteps): - model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0]) - hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state - prediction = dit_fn(hidden_states, model_time, step) - if not is_piflow: - prediction = prediction.float() - state = scheduler.step(prediction, t, state, return_dict=False)[0] - if on_step is not None: - on_step() +def _denoise_chunk(x, dit, scheduler, dit_spec, spec, step_context, on_step): + # each chunk starts at step zero and releases its GPU temporaries before the next + scheduler._step_index = None + scheduler.set_begin_index(0) + state = x[..., : dit_spec.in_visual_dim].float() + cond = x[..., dit_spec.in_visual_dim :] + frames, height, width = x.shape[1:4] + rope_pos = [ + torch.arange(frames, device=x.device), + torch.arange(height // dit_spec.patch_size[1], device=x.device), + torch.arange(width // dit_spec.patch_size[2], device=x.device), + ] + motion = ( + torch.full((1,), 900.0, device=x.device, dtype=state.dtype) + if dit_spec.use_motion_score and not spec.is_piflow + else None + ) + with bf16_autocast(x.device): + for step, t in enumerate(scheduler.timesteps): + model_time = t.to(device=x.device, dtype=torch.float32).expand( + state.shape[0] + ) + hidden_states = ( + torch.cat([state, cond], dim=-1) if cond.shape[-1] else state + ) + with step_context(step) if step_context is not None else nullcontext(): + prediction = dit( + hidden_states, + model_time, + rope_pos, + scale_factor=spec.scale_factor, + motion_score=motion, + ) + # Euler otherwise downcasts the running state to the prediction dtype + if not spec.is_piflow: + prediction = prediction.float() + state = scheduler.step(prediction, t, state, return_dict=False)[0] + if on_step is not None: + on_step() return state.float() -__all__ = [ - "DitFn", - "DitSpec", - "PiflowParams", - "SamplingSpec", - "StepCallback", - "StepContext", - "bf16_autocast", - "cast_to_module_dtype", - "check_sampler_options", - "denoise_with_scheduler", - "euler_start_timestep", - "make_dit_fn", - "module_dtype", -] +def denoise_chunks( + chunks: Sequence[torch.Tensor], + dit: Callable[..., torch.Tensor], + scheduler: Any, + *, + dit_spec: DitSpec, + spec: SamplingSpec, + device: torch.device, + step_context: Callable[[int], AbstractContextManager] | None = None, + on_step: Callable[[], None] | None = None, +) -> list[torch.Tensor]: + """Denoise CPU chunks sequentially, preserving conditioning and fp32 state.""" + if spec.is_piflow: + scheduler.set_timesteps(spec.num_steps, device=device) + else: + capped = spec.cap_noise_timestep and dit_spec.instruct_type in ( + "noise", + "hybrid", + "hybrid_anchor", + ) + start = spec.lq_noise_scale if capped else 1.0 + sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist() + results = [] + for chunk in chunks: + if not spec.is_piflow: + scheduler.set_timesteps(sigmas=sigmas, device=device) + results.append( + _denoise_chunk( + chunk.to(device), dit, scheduler, dit_spec, spec, step_context, on_step + ).cpu() + ) + return results diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/stages.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/stages.py new file mode 100644 index 0000000000..f7f842b2c8 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/stages.py @@ -0,0 +1,463 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR pipeline stages, grouped by component while retaining separate offload lifetimes.""" + +from __future__ import annotations + +import os +from functools import partial + +import torch + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( + resolve_target_hw, +) +from sglang.multimodal_gen.runtime.distributed import ( + get_decode_parallel_group_coordinator, + get_local_torch_device, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.vaes.common import ( + can_install_spatial_shard_parallel_decode, + has_decode_parallel_world, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_CHUNKS_KEY, + SR_DENOISED_KEY, + SR_DIT_SPEC_KEY, + SR_LR_LATENT_KEY, + SR_PLAN_KEY, + SR_REQUESTED_HW_KEY, + SR_SAMPLING_SPEC_KEY, + SR_TILES_KEY, + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, + build_dit_spec, + build_sampling_spec, + uses_latent_path, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + denoise_chunks, + module_dtype, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + decode_chunks, + encode_pixel_tile, + encode_video_to_lr_latent, + plan_tiles, + prepare_tile_latents, + stitch_tiles, + upscale_lr_latent_tile, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + VAE_SPATIAL_FACTOR, + pad_to_spatial_factor, + pre_upscale_video, + resolve_scale_request, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( + SOURCE_AUDIO_SAMPLE_RATE, + DecodedClip, + decode_clip, + extract_source_audio, + resize_to_target, + synthetic_clip, + to_output_video, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision import ( + precision_to_dtype, + resolve_precision, +) + +WARMUP_CLIP_FRAMES = 9 +WARMUP_CLIP_HW = (256, 384) + + +class Kandinsky6SRInputStage(PipelineStage): + """Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio. + + Record requested dimensions before VAE padding so output can remove it.""" + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale) + if batch.is_warmup: + height, width = WARMUP_CLIP_HW + clip = DecodedClip( + frames=synthetic_clip( + height=height, width=width, frames=WARMUP_CLIP_FRAMES + ), + fps=24, + source_fps=24.0, + ) + else: + path = batch.sampling_params.source_video_path() + if path is None or not os.path.isfile(path): + raise ValueError(f"Kandinsky6 SR input video not found: {path!r}") + self.log_info("Decoding %s", path) + clip = decode_clip(path) + video = clip.frames + if pre_upscale != 1.0: + video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR) + frames = video.shape[0] + # pad every scale for the VAE; output crops padding without resizing source pixels + video, (requested_h, requested_w) = pad_to_spatial_factor( + video, VAE_SPATIAL_FACTOR + ) + + batch.extra[SR_VIDEO_KEY] = video + batch.extra[SR_TILING_SCALE_KEY] = tiling_scale + batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w) + batch.fps = clip.fps + batch.num_frames = frames + batch.height = requested_h * tiling_scale + batch.width = requested_w * tiling_scale + if not batch.is_warmup: + audio = extract_source_audio( + path, + sample_rate=SOURCE_AUDIO_SAMPLE_RATE, + max_seconds=frames / clip.fps, + ) + batch.audio = torch.from_numpy(audio) if audio is not None else None + batch.audio_sample_rate = ( + SOURCE_AUDIO_SAMPLE_RATE if audio is not None else None + ) + return batch + + +class Kandinsky6SREncodeStage(PipelineStage): + """Encode the whole clip only for the LU path; pixel mode encodes tiles later.""" + + def __init__(self, vae, latent_upscaler=None) -> None: + super().__init__() + self.vae = vae + # Only consulted to pick the path (does the bank have this scale?). + self.latent_upscaler = latent_upscaler + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "vae", + phase="encode_video", + target_dtype=vae_dtype, + start_at_stage_entry=False, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + tiling_scale = batch.extra[SR_TILING_SCALE_KEY] + if not uses_latent_path(self.latent_upscaler, tiling_scale): + return batch + video = batch.extra.pop(SR_VIDEO_KEY) + with self.use_declared_component( + component_name="vae", module=self.vae, phase="encode_video" + ) as vae: + assert vae is not None + self.vae = vae + lr_latent = encode_video_to_lr_latent( + video, vae, device=get_local_torch_device() + ) + batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu() + return batch + + +class Kandinsky6SRLatentPrepStage(PipelineStage): + """LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time.""" + + def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None: + super().__init__() + self.vae = vae + self.transformer = transformer # arch-config / dtype only: never called here + self.latent_upscaler = latent_upscaler + self.scheduler = ( + scheduler # resolved sampler validation only: never stepped here + ) + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + stage_name = self._component_stage_name(stage_name) + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + uses: list[ComponentUse] = [] + if self.latent_upscaler is not None: + uses.append( + ComponentUse( + stage_name, + "latent_upscaler", + phase="upscale_tiles", + target_dtype=torch.bfloat16, + ) + ) + uses.append( + ComponentUse( + stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype + ) + ) + return uses + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + arch = server_args.pipeline_config.dit_config.arch_config + tiling_scale = batch.extra[SR_TILING_SCALE_KEY] + seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0] + spec = build_sampling_spec( + arch=arch, + tiling_scale=tiling_scale, + seed=seed, + num_steps=batch.num_inference_steps, + tiles_batch_size=batch.sr_tiles_batch_size, + tile_min_overlap=batch.sr_tile_min_overlap, + scheduler=self.scheduler, + ) + dit_spec = build_dit_spec(self.transformer) + device = get_local_torch_device() + + latent_path = uses_latent_path(self.latent_upscaler, tiling_scale) + source = batch.extra.pop(SR_LR_LATENT_KEY if latent_path else SR_VIDEO_KEY) + factor = VAE_SPATIAL_FACTOR if latent_path else 1 + plan = plan_tiles( + frame_hw=tuple(size * factor for size in source.shape[-2:]), + visual_size=spec.visual_size, + tiling_scale=tiling_scale, + tile_min_overlap=spec.tile_min_overlap, + ) + with self.use_declared_component( + component_name="latent_upscaler" if latent_path else "vae", + module=self.latent_upscaler if latent_path else self.vae, + phase="upscale_tiles" if latent_path else "encode_tiles", + ) as component: + assert component is not None + if latent_path: + self.latent_upscaler = component + tile_encoder = partial( + upscale_lr_latent_tile, + upscale_fn=partial(component.upscale, scale=tiling_scale), + lu_dtype=module_dtype(component), + scaling_factor=self.vae.scaling_factor, + device=device, + ) + else: + self.vae = component + tile_encoder = partial( + encode_pixel_tile, + vae=component, + scaling_factor=component.scaling_factor, + device=device, + ) + chunks = prepare_tile_latents( + source, + plan, + tile_encoder=tile_encoder, + latent_path=latent_path, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + batch.extra[SR_PLAN_KEY] = plan + batch.extra[SR_CHUNKS_KEY] = chunks + batch.extra[SR_SAMPLING_SPEC_KEY] = spec + batch.extra[SR_DIT_SPEC_KEY] = dit_spec + return batch + + +class Kandinsky6SRDenoisingStage(DenoisingStage): + """Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage.""" + + def _owns_compile_warmup_lifecycle(self) -> bool: + # forward wraps the tiled loop in the shared compile warmup lifecycle + return True + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Validate tile-chunk state instead of the shared text/CFG loop inputs.""" + result = VerificationResult() + result.add_check( + "extra[SR_SAMPLING_SPEC_KEY]", + batch.extra.get(SR_SAMPLING_SPEC_KEY), + V.not_none, + ) + result.add_check( + "extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none + ) + result.add_check( + "extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check( + "extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty + ) + return result + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + dit_dtype = precision_to_dtype( + server_args.pipeline_config.dit_precision, "dit_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "transformer", + phase="denoise_tiles", + target_dtype=dit_dtype, + preferred_ready_after_request=True, + memory_intensive=True, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + with self._offload_for_torch_compile_warmup(batch): + return self._denoise_tiles(batch, server_args) + + def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req: + spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY) + dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY) + chunks = batch.extra.pop(SR_CHUNKS_KEY) + device = get_local_torch_device() + + self._maybe_enable_cache_dit_and_torch_compile(spec.num_steps, batch) + + with self.use_declared_component( + component_name="transformer", module=self.transformer, phase="denoise_tiles" + ) as transformer: + assert transformer is not None + self.transformer = transformer + total = spec.num_steps * len(chunks) + with self.progress_bar( + total=total, batch=batch, desc="Kandinsky6 SR denoising" + ) as progress: + denoised = denoise_chunks( + chunks, + transformer, + self.scheduler, + dit_spec=dit_spec, + spec=spec, + device=device, + step_context=lambda step: set_forward_context( + current_timestep=step, attn_metadata=None, forward_batch=batch + ), + on_step=progress.update, + ) + self._finish_active_component_use() + batch.extra[SR_DENOISED_KEY] = denoised + return batch + + +class Kandinsky6SRDecodeStage(PipelineStage): + """VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``).""" + + def __init__(self, vae) -> None: + super().__init__() + self.vae = vae + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "vae", + phase="decode_tiles", + target_dtype=vae_dtype, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + denoised = batch.extra.pop(SR_DENOISED_KEY) + device = get_local_torch_device() + group = None + if ( + server_args.pipeline_config.vae_config.use_parallel_tiling + and has_decode_parallel_world() + and not can_install_spatial_shard_parallel_decode( + server_args.pipeline_config.vae_config + ) + ): + group = get_decode_parallel_group_coordinator() + with self.use_declared_component( + component_name="vae", module=self.vae, phase="decode_tiles" + ) as vae: + assert vae is not None + self.vae = vae + tiles = decode_chunks( + denoised, + vae, + scaling_factor=vae.scaling_factor, + device=device, + group=group, + ) + batch.extra[SR_TILES_KEY] = tiles + return batch + + +class Kandinsky6SROutputStage(PipelineStage): + """Hann-blends the decoded tiles, restores the requested output size, applies the + optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1]) + plus the mono source audio. + + CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set + to the final size, ``batch.fps`` was set by the input stage (and now also carried on + the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save + path unchanged). + """ + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: + tiles = batch.extra.pop(SR_TILES_KEY) + plan = batch.extra.pop(SR_PLAN_KEY) + video = stitch_tiles(tiles, plan) + del tiles + + requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None) + if requested_hw is not None: + requested_h, requested_w = requested_hw + video = video[ + ..., + : requested_h * plan.tiling_scale, + : requested_w * plan.tiling_scale, + ] + + target_hw = resolve_target_hw( + batch.sr_target_resolution, + tuple(video.shape[-2:]), + batch.sr_target_resize_mode, + ) + if target_hw is not None: + video = resize_to_target(video, target_hw) + batch.height, batch.width = video.shape[-2:] + return OutputBatch( + output=to_output_video(video), + audio=batch.audio, + audio_sample_rate=batch.audio_sample_rate, + fps=batch.fps, + metrics=batch.metrics, + usage=batch.usage, + ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py index 3c801b1b60..d557551282 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py @@ -6,10 +6,10 @@ Each chunk uses seed + its first tile index, matching the reference. """ from collections.abc import Callable, Iterator, Mapping, Sequence -from typing import Any import msgspec import torch +from torch.nn import functional as F from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import ( @@ -18,14 +18,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( DitSpec, SamplingSpec, - StepCallback, - StepContext, bf16_autocast, cast_to_module_dtype, - check_sampler_options, - denoise_with_scheduler, - euler_start_timestep, - make_dit_fn, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( RESOLUTIONS, @@ -36,7 +30,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k extract_all_tiles, latent_tile_grid_from_pixel_grid, stitch_tiles_hanning, - upsample_tiles_to_base, ) LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor] @@ -147,188 +140,57 @@ def encode_pixel_tile( return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor -def build_chunk_latent( - lq_tiles: Sequence[torch.Tensor], - *, - seed: int, - dit_spec: DitSpec, - spec: SamplingSpec, - device: torch.device, -) -> torch.Tensor: - """Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles.""" - lq = torch.cat(list(lq_tiles), dim=0) - batch = len(lq_tiles) - x = build_initial_latent( - instruct_type=dit_spec.instruct_type, - visual_cond=dit_spec.visual_cond, - in_visual_dim=dit_spec.in_visual_dim, - lq_latent=lq, - batch_size=batch, - duration=lq.shape[0] // batch, - height=lq.shape[1], - width=lq.shape[2], - device=device, - seed=seed, - lq_noise_scale=spec.lq_noise_scale, - lq_noise_type=spec.lq_noise_type, - lq_channel_noise_scale=spec.lq_channel_noise_scale, - # The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before - # building the initial latent), not a hard-coded fp32 regardless of precision. - dtype=dit_spec.dtype, - ) - return x.reshape(batch, -1, *x.shape[1:]) - - @torch.no_grad() -def prepare_lu_tile_latents( - lr_latent: torch.Tensor, +def prepare_tile_latents( + source: torch.Tensor, plan: TilePlan, *, - upscale_fn: LatentUpscaleFn, - lu_dtype: torch.dtype | None, - scaling_factor: float, + tile_encoder: Callable[[torch.Tensor], torch.Tensor], + latent_path: bool, dit_spec: DitSpec, spec: SamplingSpec, device: torch.device, spatial_factor: int = VAE_SPATIAL_FACTOR, ) -> list[torch.Tensor]: - """LU path: initial latents (CPU) of every chunk from the whole-video latent.""" - latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor) - tiles = extract_all_tiles(lr_latent, latent_grid) - chunks: list[torch.Tensor] = [] - for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size): - lq_tiles = [ - upscale_lr_latent_tile( - tile, - upscale_fn, - lu_dtype=lu_dtype, - scaling_factor=scaling_factor, - device=device, - ) - for tile in tiles[start:stop] - ] - chunk = build_chunk_latent( - lq_tiles, - seed=spec.seed + start, - dit_spec=dit_spec, - spec=spec, - device=device, - ) - chunks.append(chunk.cpu()) - return chunks - - -@torch.no_grad() -def prepare_pixel_tile_latents( - video: torch.Tensor, - plan: TilePlan, - *, - vae: torch.nn.Module, - scaling_factor: float, - dit_spec: DitSpec, - spec: SamplingSpec, - device: torch.device, -) -> list[torch.Tensor]: - """Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk.""" - tiles = extract_all_tiles(video, plan.pixel_grid) - chunks: list[torch.Tensor] = [] - for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size): - enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw) - lq_tiles = [ - encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device) - for tile in enlarged - ] - chunk = build_chunk_latent( - lq_tiles, - seed=spec.seed + start, - dit_spec=dit_spec, - spec=spec, - device=device, - ) - chunks.append(chunk.cpu()) - return chunks - - -def reset_scheduler_for_chunk(scheduler: Any) -> None: - """Reset step and begin indices so each tile chunk starts at the same timestep.""" - scheduler._step_index = None - scheduler.set_begin_index(0) - - -@torch.no_grad() -def denoise_chunk( - x: torch.Tensor, - dit: Callable[..., torch.Tensor], - scheduler: Any, - *, - dit_spec: DitSpec, - spec: SamplingSpec, - step_context: StepContext | None = None, - on_step: StepCallback | None = None, -) -> torch.Tensor: - """Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler.""" - is_piflow = spec.piflow is not None - check_sampler_options( - piflow=spec.piflow, - cap_noise_timestep=spec.cap_noise_timestep, - instruct_type=dit_spec.instruct_type, + """Encode/upscale one tile batch at a time; keep prepared chunks on CPU.""" + grid = ( + latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor) + if latent_path + else plan.pixel_grid ) - if not is_piflow: - # The Euler start timestep is only meaningful for the scheduler this chunk is about to - # run: re-derive it instead of baking a stale one into ``set_timesteps`` up front. - start = euler_start_timestep( - cap_noise_timestep=spec.cap_noise_timestep, - lq_noise_scale=spec.lq_noise_scale, + tiles = extract_all_tiles(source, grid) + chunks = [] + for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size): + inputs = tiles[start:stop] + if not latent_path: + inputs = [ + F.interpolate( + tile.float(), + size=plan.base_hw, + mode="bilinear", + align_corners=False, + ).permute(0, 2, 3, 1) + for tile in inputs + ] + lq_tiles = [tile_encoder(tile) for tile in inputs] + lq = torch.cat(lq_tiles, dim=0) + x = build_initial_latent( instruct_type=dit_spec.instruct_type, + visual_cond=dit_spec.visual_cond, + in_visual_dim=dit_spec.in_visual_dim, + lq_latent=lq, + device=device, + seed=spec.seed + start, + lq_noise_scale=spec.lq_noise_scale, + lq_noise_type=spec.lq_noise_type, + lq_channel_noise_scale=spec.lq_channel_noise_scale, + # conditioning noise must round in the loaded DiT dtype + dtype=dit_spec.dtype, ) - sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist() - scheduler.set_timesteps(sigmas=sigmas, device=x.device) - reset_scheduler_for_chunk(scheduler) - dit_fn = make_dit_fn( - dit, - latent_frames_hw=tuple(x.shape[1:4]), - patch_size=dit_spec.patch_size, - scale_factor=spec.scale_factor, - use_motion_score=dit_spec.use_motion_score and not is_piflow, - step_context=step_context, - ) - with bf16_autocast(x.device): - return denoise_with_scheduler( - x, - dit_fn, - scheduler, - channels=dit_spec.in_visual_dim, - is_piflow=is_piflow, - on_step=on_step, - ) - - -def denoise_chunks( - chunks: Sequence[torch.Tensor], - dit: Callable[..., torch.Tensor], - scheduler: Any, - *, - dit_spec: DitSpec, - spec: SamplingSpec, - device: torch.device, - step_context: StepContext | None = None, - on_step: StepCallback | None = None, -) -> list[torch.Tensor]: - """Denoise chunks sequentially on device and return CPU latents.""" - if spec.piflow is not None: - scheduler.set_timesteps(spec.piflow.nfe, device=device) - return [ - denoise_chunk( - chunk.to(device), - dit, - scheduler, - dit_spec=dit_spec, - spec=spec, - step_context=step_context, - on_step=on_step, - ).cpu() - for chunk in chunks - ] + chunk = x.reshape(len(lq_tiles), -1, *x.shape[1:]) + del lq, x + chunks.append(chunk.cpu()) + return chunks @torch.no_grad() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py index f523d225a0..2a620d0254 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py @@ -58,23 +58,6 @@ def extract_all_tiles( return tiles -def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """Normalize Hann-weighted tiles without flooring positive weights. - - Corner weights are tiny but valid; clamping them darkens tile boundaries. - Pixels outside all tiles have zero weight and remain zero.""" - covered = weight > 0 - safe_weight = torch.where(covered, weight, torch.ones_like(weight)) - return torch.where(covered, acc / safe_weight, torch.zeros_like(acc)) - - -def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor: - """Create a [h, w] Hann window with nonzero endpoints.""" - wy = torch.hann_window(h + 2, device=device)[1:-1] - wx = torch.hann_window(w + 2, device=device)[1:-1] - return wy[:, None] * wx[None, :] - - def stitch_tiles_hanning( tiles: list[torch.Tensor], grid: TileGrid, @@ -88,8 +71,11 @@ def stitch_tiles_hanning( hr_tile_h, hr_tile_w = first.shape[2], first.shape[3] device = first.device - window = hanning_window_2d(hr_tile_h, hr_tile_w, device) - window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w] + # nonzero endpoints keep the outermost tile pixels covered + wy = torch.hann_window(hr_tile_h + 2, device=device)[1:-1] + wx = torch.hann_window(hr_tile_w + 2, device=device)[1:-1] + window = (wy[:, None] * wx[None, :])[None, None] + del wy, wx out_h = original_h * scale out_w = original_w * scale @@ -106,7 +92,10 @@ def stitch_tiles_hanning( weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window tile_idx += 1 - return _normalize_blend(pred_acc, weight_acc) + # preserve tiny positive corner weights; only uncovered pixels stay zero + covered = weight_acc > 0 + safe_weight = torch.where(covered, weight_acc, torch.ones_like(weight_acc)) + return torch.where(covered, pred_acc / safe_weight, torch.zeros_like(pred_acc)) def axis_positions_even( @@ -179,18 +168,6 @@ def closest_base_resolution( return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio)) -def upsample_tiles_to_base( - raw_tiles: list[torch.Tensor], base_h: int, base_w: int -) -> list[torch.Tensor]: - """Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``.""" - return [ - functional.interpolate( - tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False - ).permute(0, 2, 3, 1) - for tile in raw_tiles - ] - - def resolve_scale_request(scale: float) -> tuple[int, float]: """Decompose total scale into (integer tiling scale, pixel pre-upscale).""" requested = float(scale) @@ -240,9 +217,3 @@ def pad_to_spatial_factor( return video, (height, width) padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate") return padded, (height, width) - - -def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor: - """Crop [..., H, W] to hw from the top-left, undoing bottom/right padding.""" - height, width = hw - return video[..., :height, :width] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py index 38bdb84299..76c77b7d81 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py @@ -1,24 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 -"""Streaming video / audio input and output helpers of Kandinsky 6 video SR. +# Frame selection and resize adapted from the Kandinsky 6 SR inference reference +# (k6_video / sr_core, Apache-2.0). +"""Bounded-memory SR video/audio I/O, preserving reference frame selection and resize.""" -Decoding follows the reference CLI (``read_video_tchw_uint8`` -> -``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams: -only the frames that survive fps resampling and the 121-frame cap are converted to RGB -and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily -(it is part of the ``diffusion`` extra). -""" +import logging import msgspec import numpy as np import torch +from torch.nn import functional as F -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( - MAX_NUM_FRAMES, - RESAMPLE_FPS_TOLERANCE, - TARGET_FPS, - clip_to_aligned_frames, +from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( + ASPECT_MISMATCH_TOLERANCE, ) +logger = logging.getLogger(__name__) +# trained on 5 s at 24 fps; select at most 121 frames, aligned to 1+8k +TARGET_FPS = 24 +MAX_NUM_FRAMES = 121 +RESAMPLE_FPS_TOLERANCE = 1.5 SOURCE_AUDIO_SAMPLE_RATE = 44100 OUTPUT_FRAME_CHUNK = 8 @@ -32,16 +32,10 @@ class DecodedClip(msgspec.Struct, frozen=True): class FrameSelector: - """Streaming form of ``resample_to_target_fps`` + the first-121-frames cap. + """Stream reference indices round(i*step), capped at int(total/step) and max_frames. - Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the - selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further - frame can change the result, and :meth:`limit` trims the selection once the source - length is known. For a source faster than the target the selected indices are - ``round(i * step)``, ``step = source_fps / target_fps``, of which there are - ``int(total / step)`` (those below ``total``), exactly as in the reference; - otherwise every frame is selected. - """ + In-tolerance and slower sources keep every frame. Stop only once the source + length guarantees that no retained frame will be trimmed by limit().""" def __init__( self, @@ -116,9 +110,9 @@ def decode_clip( kept = selector.limit(seen, len(frames)) if kept == 0: raise ValueError(f"Video {path} has no readable frames.") - video = torch.stack(frames[:kept]).contiguous() + kept = 1 + 8 * ((kept - 1) // 8) return DecodedClip( - frames=clip_to_aligned_frames(video, max_frames), + frames=torch.stack(frames[:kept]).contiguous(), fps=selector.effective_fps, source_fps=source_fps, ) @@ -188,3 +182,48 @@ def to_output_video(video: torch.Tensor) -> torch.Tensor: (chunk.float() + 0.5) / 255.0 ).to(torch.float16) return out + + +def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor: + """Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping.""" + channels, frames, height, width = video.shape + target_h, target_w = target_hw + if (height, width) == (target_h, target_w): + return video + if target_h > height or target_w > width: + raise ValueError( + f"target {target_w}x{target_h} exceeds the SR result {width}x{height}; " + "pick a lower tier or a higher --resolution-scale for this source." + ) + source_ratio, target_ratio = width / height, target_w / target_h + if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE: + logger.warning( + "Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f; resizing " + "anisotropically (no crop), the picture will be squeezed by %.1f%%.", + width, + height, + source_ratio, + target_w, + target_h, + target_ratio, + abs(source_ratio / target_ratio - 1) * 100, + ) + out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8) + for start in range(0, frames, OUTPUT_FRAME_CHUNK): + chunk = video[:, start : start + OUTPUT_FRAME_CHUNK].permute(1, 0, 2, 3).float() + resized = F.interpolate( + chunk, size=target_hw, mode="bilinear", antialias=True, align_corners=False + ) + out[:, start : start + OUTPUT_FRAME_CHUNK] = ( + resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3) + ) + logger.info( + "Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)", + width, + height, + target_w, + target_h, + width / target_w, + height / target_h, + ) + return out diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py deleted file mode 100644 index dc466b53fe..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Vendored from sr_core (parity-tested against the k6_video reference); adapted from -# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0). -"""Frame selection and bounded-memory output resize for Kandinsky 6 SR.""" - -from __future__ import annotations - -import logging - -import torch -from torch.nn import functional as f # noqa: N812 - -from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( - ASPECT_MISMATCH_TOLERANCE, -) - -logger = logging.getLogger(__name__) - -# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) --- -# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames -# (= 1 + 8*15); feeding more frames or another rate is out of contract. -TARGET_FPS: int = 24 -MAX_NUM_FRAMES: int = 121 -# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no -# resample): covers 23.98 (NTSC) and 25.00 sources. -RESAMPLE_FPS_TOLERANCE: float = 1.5 -# Frames per interpolate call when downscaling the output (a 2K clip is -# ~2.4 GB as uint8, so the resize walks the clip in chunks). -RESIZE_FRAME_CHUNK: int = 8 - - -def select_frame_indices( - total_frames: int, src_fps: float, target_fps: float -) -> list[int]: - """Select rounded frame indices to downsample src_fps to target_fps.""" - step = src_fps / target_fps - indices = [round(i * step) for i in range(int(total_frames / step))] - return [i for i in indices if i < total_frames] - - -def resample_to_target_fps( - video: torch.Tensor, - src_fps: float, - target_fps: int = TARGET_FPS, -) -> tuple[torch.Tensor, int]: - """Downsample high-fps inputs; retain near-target and lower-fps inputs. - - Return [T, C, H, W] video and its effective playback rate.""" - if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE: - return video, round(src_fps) - if src_fps > target_fps: - indices = select_frame_indices(video.shape[0], src_fps, target_fps) - logger.warning( - "Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); " - "source temporal detail beyond %sfps is discarded.", - src_fps, - target_fps, - video.shape[0], - len(indices), - target_fps, - ) - return video[indices], target_fps - logger.warning( - "Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); " - "output is mildly out of distribution.", - src_fps, - target_fps, - ) - return video, round(src_fps) - - -def clip_to_aligned_frames( - video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES -) -> torch.Tensor: - """Cap the frame count, then floor-align to 1+8k; reject empty video.""" - if max_num_frames > 0: - video = video[:max_num_frames] - aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0 - if aligned == 0: - msg = "Video has no readable frames." - raise ValueError(msg) - return video[:aligned] - - -def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor: - """Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping.""" - _c, _t, height, width = video.shape - target_h, target_w = target_hw - if (height, width) == (target_h, target_w): - return video - if target_h > height or target_w > width: - msg = ( - f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would " - f"upscale. Pick a lower tier or a higher --resolution-scale for this source." - ) - raise ValueError(msg) - - source_ratio, target_ratio = width / height, target_w / target_h - if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE: - logger.warning( - "Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically " - "(no crop), the picture will be squeezed by %.1f%%.", - width, - height, - source_ratio, - target_w, - target_h, - target_ratio, - abs(source_ratio / target_ratio - 1) * 100, - ) - - channels, frames = video.shape[0], video.shape[1] - out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8) - for start in range(0, frames, RESIZE_FRAME_CHUNK): - chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float() - resized = f.interpolate( - chunk, - size=(target_h, target_w), - mode="bilinear", - antialias=True, - align_corners=False, - ) - out[:, start : start + RESIZE_FRAME_CHUNK] = ( - resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3) - ) - - logger.info( - "Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)", - width, - height, - target_w, - target_h, - width / target_w, - height / target_h, - ) - return out diff --git a/python/sglang/multimodal_gen/test/unit/conftest.py b/python/sglang/multimodal_gen/test/unit/conftest.py index 25bccd9fb5..ed249b90af 100644 --- a/python/sglang/multimodal_gen/test/unit/conftest.py +++ b/python/sglang/multimodal_gen/test/unit/conftest.py @@ -5,7 +5,28 @@ from types import SimpleNamespace import pytest from sglang.multimodal_gen.runtime import server_args as server_args_module +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) from sglang.multimodal_gen.runtime.server_args import set_global_server_args +from sglang.srt.utils.network import get_free_port_below_ephemeral + + +@pytest.fixture(scope="module") +def single_process_model_parallel(): + with pytest.MonkeyPatch.context() as patch: + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(get_free_port_below_ephemeral()), + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + patch.setenv(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + yield def _make_unit_server_args(): diff --git a/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py index 70167e2fa0..0c069c49fd 100644 --- a/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py +++ b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py @@ -3,6 +3,7 @@ import copy from collections.abc import Callable, Mapping, Sequence +from functools import partial from types import SimpleNamespace from typing import Any @@ -21,6 +22,7 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( Kandinsky6SRSamplingParams, ) from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( Kandinsky6SRTransformer3DModel, ) @@ -37,29 +39,30 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( DitSpec, SamplingSpec, + denoise_chunks, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( LatentUpscaleFn, decode_chunks, - denoise_chunks, + encode_pixel_tile, encode_video_to_lr_latent, plan_tiles, - prepare_lu_tile_latents, - prepare_pixel_tile_latents, + prepare_tile_latents, stitch_tiles, + upscale_lr_latent_tile, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( RESOLUTIONS, VAE_SPATIAL_FACTOR, ) -# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels. +# official transformer layout: total output width, scheduler configured separately TINY_DIT = dict( in_visual_dim=4, in_text_dim=8, in_text_dim2=8, time_dim=16, - out_visual_dim=4, + out_visual_dim=12, patch_size=[1, 2, 2], model_dim=32, ff_dim=64, @@ -69,15 +72,16 @@ TINY_DIT = dict( visual_cond=False, instruct_type="noise", use_text=False, + attribute_overrides=None, + sr_params=dict(visual_size=[512], scale_factor={"512": [1.0, 1.0, 1.0]}), +) +TINY_PIFLOW = dict( n_grid=3, - piflow_nfe=2, - piflow_num_policy_substeps=8, - piflow_final_step_size_scale=0.5, - piflow_shift=5.0, - piflow_eps=1e-6, - attribute_overrides={"instruct_type": "noise"}, - sr_visual_size=[512], - sr_scale_factor={"512": [1.0, 1.0, 1.0]}, + nfe=2, + num_policy_substeps=8, + final_step_size_scale=0.5, + shift=5.0, + eps=1e-6, ) TINY_KVAE = dict( ch=8, @@ -211,9 +215,7 @@ def super_resolve( ) -> torch.Tensor: """Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``. - ``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the - already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded - component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits. + ``upscale_fn`` selects the LU path; without it the pixel path runs. """ device = torch.device(device) common = dict( @@ -230,30 +232,43 @@ def super_resolve( lr_latent.shape[-1] * spatial_factor, ) plan = plan_tiles(frame_hw=frame_hw, **common) - chunks = prepare_lu_tile_latents( - lr_latent, - plan, + source = lr_latent + tile_encoder = partial( + upscale_lr_latent_tile, upscale_fn=upscale_fn, lu_dtype=lu_dtype, scaling_factor=scaling_factor, - dit_spec=dit_spec, - spec=spec, device=device, - spatial_factor=spatial_factor, ) else: plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common) - chunks = prepare_pixel_tile_latents( - video, - plan, + source = video + tile_encoder = partial( + encode_pixel_tile, vae=vae, scaling_factor=scaling_factor, - dit_spec=dit_spec, - spec=spec, device=device, ) + chunks = prepare_tile_latents( + source, + plan, + tile_encoder=tile_encoder, + latent_path=upscale_fn is not None, + dit_spec=dit_spec, + spec=spec, + device=device, + spatial_factor=spatial_factor, + ) denoised = denoise_chunks( - chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device + chunks, + dit, + scheduler, + dit_spec=dit_spec, + spec=spec, + device=device, + step_context=lambda step: set_forward_context( + current_timestep=step, attn_metadata=None + ), ) tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device) return stitch_tiles(tiles, plan) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py index 55ebab7d46..6a2846208c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py @@ -15,6 +15,7 @@ reference). from __future__ import annotations +import pytest import torch from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( @@ -78,12 +79,16 @@ def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load(): vae = Kandinsky6AudioVAE(config) assert vae.vae._weights_normalized is True - # `load_state_dict(vae.state_dict())` mirrors VAELoader's real call - # (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's - # own current weights standing in for "a checkpoint's own weights" -- - # exactly what happens when a real checkpoint's stored tensors are - # loaded: no code path scales or otherwise disturbs them afterward. - vae.load_state_dict(vae.state_dict()) + checkpoint = dict(vae.state_dict()) + checkpoint["vae.data_mean"] = torch.full((1, 80, 1), -2.5) + checkpoint["vae.data_std"] = torch.full((1, 80, 1), 1.25) + vae.load_state_dict(checkpoint, strict=True) + torch.testing.assert_close(vae.vae.data_mean, checkpoint["vae.data_mean"]) + torch.testing.assert_close(vae.vae.data_std, checkpoint["vae.data_std"]) + for key in ("vae.data_mean", "vae.data_std"): + incomplete = {name: value for name, value in checkpoint.items() if name != key} + with pytest.raises(RuntimeError, match=key): + vae.load_state_dict(incomplete, strict=True) latents = torch.randn(1, vae.vae.embed_dim, 4) mel_a = vae.decode(latents.clone()) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py index a0b4f9c16c..5f13dfccdf 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py @@ -4,7 +4,6 @@ from __future__ import annotations import importlib.util -import os from types import SimpleNamespace import pytest @@ -23,10 +22,6 @@ from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( refresh_context_on_transformer, ) from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) from sglang.multimodal_gen.runtime.layers.attention.selector import ( global_force_attn_backend_context_manager, ) @@ -45,22 +40,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.server_args import get_global_server_args from sglang.multimodal_gen.runtime.utils import precision - -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - # Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which - # need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's - # fixture of the same name. - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29513", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +pytestmark = pytest.mark.usefixtures("single_process_model_parallel") def _randomize(module: torch.nn.Module, seed: int) -> None: @@ -70,29 +50,18 @@ def _randomize(module: torch.nn.Module, seed: int) -> None: param.copy_(torch.randn(param.shape, generator=generator) * 0.05) -def test_time_embeddings_forward_under_bf16_linear_weights(): - # Minimal, pinpoint reproduction of the review comment: a bf16-weight - # Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not - # raise, and must return bf16 (the Linear weights' dtype). - module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8) +@pytest.mark.parametrize( + "modulation", [False, True], ids=["time-embedding", "modulation"] +) +def test_bf16_projection_accepts_fp32_conditioning(modulation): + module = ( + Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9) + if modulation + else Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8) + ) _randomize(module, seed=0) - module = module.to(torch.bfloat16) - - out = module(torch.tensor([0.0, 500.0, 999.0])) - - assert out.dtype == torch.bfloat16 - assert torch.isfinite(out.float()).all() - - -def test_modulation_forward_under_bf16_linear_weights(): - # Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the - # identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time - # embedding with bf16 weights, it must not raise either. - module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9) - module = module.to(torch.bfloat16) - - out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to - + inputs = torch.randn(2, 8) if modulation else torch.tensor([0.0, 500.0, 999.0]) + out = module.to(torch.bfloat16)(inputs) assert out.dtype == torch.bfloat16 assert torch.isfinite(out.float()).all() @@ -297,7 +266,6 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_ t_expand=torch.tensor([500.0], device="cuda"), visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)], scale_factor=(1.0, 1.0, 1.0), - sparse_params=None, visual_token_type_ids=None, ) @@ -360,71 +328,10 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_ runner.reset() -@pytest.mark.skipif( - not torch.cuda.is_available(), - reason=( - "the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real " - "CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 " - "numerics are only meaningful on real hardware anyway" - ), -) -def test_kandinsky6_dit_real_bf16_forward_does_not_crash(): - """Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose - weights are bf16 -- the exact configuration the SGLang PR review comment flagged - as broken (a fp32 sinusoidal embedding feeding a bf16 Linear).""" - device = torch.device("cuda") - config = Kandinsky6VideoAudioConfig() - arch_kwargs = _tiny_multimodal_arch_kwargs() - config.update_model_arch(dict(arch_kwargs)) - dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval() - _randomize(dit, seed=1) - dit = dit.to(device=device, dtype=torch.bfloat16) - - batch, duration, height, width = 1, 2, 2, 2 - text_len, audio_len = 3, 5 - hidden_states = torch.randn( - batch, duration, height, width, 4, dtype=torch.bfloat16, device=device - ) - hidden_states_audio = torch.randn( - batch, audio_len, 4, dtype=torch.bfloat16, device=device - ) - encoder_hidden_states = torch.randn( - batch, text_len, 8, dtype=torch.bfloat16, device=device - ) - pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device) - # Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the - # caller), same as every real sampling loop -- exactly the dtype that used to - # mismatch the bf16 Linear. - timestep = torch.tensor([500.0], device=device) - visual_rope_pos = ( - torch.arange(duration, device=device), - torch.arange(height, device=device), - torch.arange(width, device=device), - ) - text_rope_pos = torch.arange(text_len, device=device) - - video_out, audio_out = dit( - hidden_states=hidden_states, - hidden_states_audio=hidden_states_audio, - encoder_hidden_states=encoder_hidden_states, - timestep=timestep, - pooled_projections=pooled_projections, - visual_rope_pos=visual_rope_pos, - text_rope_pos=text_rope_pos, - ) - - assert video_out.shape == (batch, duration, height, width, 4) - assert audio_out.shape == (batch, audio_len, 4) - assert video_out.dtype == torch.bfloat16 - assert audio_out.dtype == torch.bfloat16 - assert torch.isfinite(video_out.float()).all() - assert torch.isfinite(audio_out.float()).all() - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("separate_cfg", [False, True]) @torch.no_grad() -def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg): +def test_bf16_joint_forward_and_cache_dit_lifetime(separate_cfg): config = Kandinsky6VideoAudioConfig() arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4} config.update_model_arch(arch) @@ -445,18 +352,19 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg): outputs = [] for step in range(3): for branch in range(2 if separate_cfg else 1): - outputs.append( - dit( - **( - inputs - | { - "pooled_projections": inputs["pooled_projections"] - + branch - } - ), - timestep=torch.tensor([500.0 - step * 100], device="cuda"), + with set_forward_context(current_timestep=step, attn_metadata=None): + outputs.append( + dit( + **( + inputs + | { + "pooled_projections": inputs["pooled_projections"] + + branch + } + ), + timestep=torch.tensor([500.0 - step * 100], device="cuda"), + ) ) - ) return outputs def assert_exact(actual, expected): @@ -465,6 +373,12 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg): torch.testing.assert_close(tensor, target, rtol=0, atol=0) reference = run_steps() + for video, audio in reference: + assert video.shape == inputs["hidden_states"].shape + assert audio.shape == inputs["hidden_states_audio"].shape + for tensor in (video, audio): + assert tensor.dtype == torch.bfloat16 + assert torch.isfinite(tensor).all() enable_cache_on_transformer( dit, CacheDitConfig( diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py deleted file mode 100644 index 6fdc167d7d..0000000000 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor -must aspect-preserving-resize-then-centre-crop, matching the diffusers -reference's ``encode_i2va_first_frame`` -(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height, -src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to -``(height, width)``), not stretch-resize the whole source image to the -target box (which distorts its aspect ratio whenever the source doesn't -already match the target aspect ratio). -""" - -from __future__ import annotations - -import numpy as np -import PIL.Image -import pytest -import torch - -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import ( - Kandinsky6ImageEncodingStage, -) - - -def _gradient_image(width: int, height: int) -> PIL.Image.Image: - """A deterministic, per-pixel-distinguishable RGB image (not a flat - color), so resize/crop geometry is actually verifiable pixel-for-pixel. - """ - xs = np.linspace(0, 255, width, dtype=np.uint8) - ys = np.linspace(0, 255, height, dtype=np.uint8) - grid = np.zeros((height, width, 3), dtype=np.uint8) - grid[..., 0] = xs[None, :] - grid[..., 1] = ys[:, None] - grid[..., 2] = 128 - return PIL.Image.fromarray(grid, mode="RGB") - - -@pytest.mark.parametrize( - "src_h,src_w,height,width", - [ - (1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case) - (1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case) - (1024, 1536, 512, 768), # already 3:2 -- matching aspect - ], -) -def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width): - # encode_i2va_first_frame: scale = min(src_h/height, src_w/width); - # resize to (int(src_h/scale), int(src_w/scale)), then centre-crop. - scale = min(src_h / height, src_w / width) - expected = (int(src_h / scale), int(src_w / scale)) - assert ( - Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width) - == expected - ) - # The resize always covers the target box on both axes (never smaller). - assert expected[0] >= height - assert expected[1] >= width - - -@pytest.mark.parametrize( - "src_w,src_h,min_mean_abs_diff", - [ - (1920, 1080, 0.015), # 16:9 landscape source (audit's own case) - (1080, 1920, 0.05), # 9:16 portrait source (audit's own case) - ], -) -def test_mismatched_aspect_pil_image_is_cropped_not_stretched( - src_w, src_h, min_mean_abs_diff -): - # The model's documented default target resolution (512x768, 2:3) - # against realistic phone/webcam source aspect ratios. - height, width = 512, 768 - image = _gradient_image(src_w, src_h) - - produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width) - assert produced.shape == (1, 3, height, width) - - # Independent oracle: the diffusers reference's own aspect-preserving - # resize + centre-crop, computed directly via PIL, without calling any - # of the code under test. - scale = min(src_h / height, src_w / width) - new_h, new_w = int(src_h / scale), int(src_w / scale) - resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR) - top, left = (new_h - height) // 2, (new_w - width) // 2 - cropped = resized.crop((left, top, left + width, top + height)) - expected_np = np.array(cropped).astype(np.float32) / 255.0 - expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0 - - assert torch.allclose(produced[0], expected, atol=2.0 / 255.0) - - # The deleted stretch-resize procedure (plain `.resize((width, height))`, - # no crop) produces a materially different image whenever the source - # aspect ratio doesn't already match the target -- proving this is a - # real behavior change, not a no-op refactor. - stretched = image.resize((width, height), resample=PIL.Image.BILINEAR) - stretched_np = np.array(stretched).astype(np.float32) / 255.0 - stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0 - mean_abs_diff = (produced[0] - stretched_t).abs().mean().item() - assert mean_abs_diff > min_mean_abs_diff - - -def test_matching_aspect_pil_image_resize_only_agrees_with_stretch(): - # A sanity check on the oracle itself: when the source already matches - # the target aspect ratio, cover-resize-then-crop and plain stretch - # resize agree closely (no crop actually removes anything) -- so the - # large the divergence asserted above is specifically an aspect-ratio - # effect, not a general property of the two code paths. - height, width = 64, 96 - src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3... - # (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2) - image = _gradient_image(src_w, src_h) - - produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width) - stretched = image.resize((width, height), resample=PIL.Image.BILINEAR) - stretched_np = np.array(stretched).astype(np.float32) / 255.0 - stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0 - - mean_abs_diff = (produced[0] - stretched_t).abs().mean().item() - assert mean_abs_diff < 0.02 - - -def test_tensor_input_uses_cover_crop_geometry_too(): - height, width = 64, 96 - src_h, src_w = 108, 192 - tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1] - - produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width) - assert produced.shape[-2:] == (height, width) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py index d97157e755..5e27f81017 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py @@ -19,10 +19,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( Kandinsky6TI2VAPipelineConfig, ) -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) from sglang.multimodal_gen.runtime.layers.attention.selector import ( global_force_attn_backend_context_manager, ) @@ -47,24 +43,12 @@ from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import ( TINY_KVAE, TINY_LU_MODEL, ) -from sglang.srt.utils.network import get_free_port_below_ephemeral pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.fixture(autouse=True) -def single_gpu(monkeypatch, default_global_server_args): - if not model_parallel_is_initialized(): - port = get_free_port_below_ephemeral() - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT=str(port), - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - monkeypatch.setenv(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +def single_gpu(monkeypatch, default_global_server_args, single_process_model_parallel): monkeypatch.setattr( precision._mixed_precision_state, "state", diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py index 3a25fe270f..eb8ee57e3a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py @@ -84,15 +84,6 @@ def _make_pipeline() -> Kandinsky6TI2VAPipeline: return pipeline -def test_pipeline_identity_and_required_modules(): - assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline" - assert Kandinsky6TI2VAPipeline.is_video_pipeline is True - assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES - # No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder - # and the BigVGAN vocoder in one checkpoint component. - assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules - - def test_disagg_role_rejects_non_monolithic_deployment(): pipeline = object.__new__(Kandinsky6TI2VAPipeline) @@ -107,17 +98,16 @@ def test_disagg_role_rejects_non_monolithic_deployment(): def test_create_pipeline_stages_produces_documented_order(): pipeline = _make_pipeline() - server_args = MagicMock() - - Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args) - - actual_types = [type(stage) for stage in pipeline.stages] - expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER] - assert actual_types == expected_types - - actual_names = list(pipeline._stage_name_mapping.keys()) - expected_names = [name for name, _ in EXPECTED_STAGE_ORDER] - assert actual_names == expected_names + assert pipeline.pipeline_name == "Kandinsky6TI2VAPipeline" + assert pipeline.is_video_pipeline is True + assert pipeline._required_config_modules == EXPECTED_REQUIRED_MODULES + pipeline.create_pipeline_stages(MagicMock()) + assert [type(stage) for stage in pipeline.stages] == [ + stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER + ] + assert [ + (name, type(stage)) for name, stage in pipeline._stage_name_mapping.items() + ] == EXPECTED_STAGE_ORDER # get_module wiring: each stage received the mocked module it asked for. denoising_stage = pipeline._stage_name_mapping["denoising_stage"] @@ -224,7 +214,6 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch): t_expand=torch.zeros(1), visual_rope_pos=[], scale_factor=(1.0, 1.0, 1.0), - sparse_params=None, visual_token_type_ids=None, ) for result, reference in zip(actual, expected, strict=True): @@ -237,10 +226,7 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch): def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler( guidance_scale, ): - """The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance, - so the run must not silently ignore a guidance the caller asked for; the - Diffusers pipeline raises the same way ("PiflowScheduler requires - guidance_weight=1.0").""" + """The distilled checkpoint must reject unsupported CFG, not silently ignore it.""" scheduler = _pro_distill_scheduler() batch = SimpleNamespace( timesteps=scheduler.timesteps, @@ -272,11 +258,6 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance( text_lens = [] class _FakeDiT(torch.nn.Module): - """A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage`` - now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks - ``self.transformer.modules()`` to infer the attention backend -- a plain - callable has no ``.modules()``.""" - def forward(self, **kwargs): assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0) text_lens.append(kwargs["encoder_hidden_states"].shape[1]) @@ -309,28 +290,13 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance( is_warmup=True, sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"), ) - arch = SimpleNamespace( - attention_engine="auto", - in_visual_dim=video_channels, - patch_size=(1, 2, 2), - scale_factor=(1.0, 2.0, 2.0), - ) - pipeline_config = SimpleNamespace( - dit_config=SimpleNamespace(arch_config=arch), - dit_precision="fp32", - vae_config=SimpleNamespace( - arch_config=SimpleNamespace(spatial_compression_ratio=8) - ), - get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0], - # CFG is disabled in this test (guidance_scale=1.0, - # do_classifier_free_guidance=False), so the policy never needs a - # negative branch, but Kandinsky6DenoisingStage still reads - # `cfg_policy` / `get_classifier_free_guidance_scale` off the config - # unconditionally to build the (single-branch) CFGPolicy. - cfg_policy=CFGPolicy(), - get_classifier_free_guidance_scale=lambda _batch, guidance_scale: ( - guidance_scale - ), + pipeline_config = Kandinsky6TI2VAPipelineConfig(dit_precision="fp32") + pipeline_config.dit_config.update_model_arch( + dict( + in_visual_dim=video_channels, + patch_size=(1, 2, 2), + scale_factor=(1.0, 2.0, 2.0), + ) ) server_args = SimpleNamespace( pipeline_config=pipeline_config, enable_cfg_parallel=False diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py index 11a24e28b9..77c86fb27f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py @@ -120,4 +120,4 @@ def test_gate_sum_preserves_fp32_intermediates(): gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384] expected = (x.float() + gate.float() * update.float()).to(x.dtype) assert not torch.equal(expected, x + gate * update) - assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected) + assert torch.equal(kandinsky6.residual_gate_fp32(x, update, gate), expected) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py index fa898bb575..d9a3186427 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py @@ -5,6 +5,7 @@ import argparse import re import pytest +from kandinsky6_sr_tiny_components import TINY_DIT from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( Kandinsky6SRArchConfig, @@ -25,85 +26,10 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping -TINY_DIT = dict( - in_visual_dim=4, - in_text_dim=8, - in_text_dim2=8, - time_dim=16, - out_visual_dim=4, - patch_size=[1, 2, 2], - model_dim=32, - ff_dim=64, - num_text_blocks=0, - num_visual_blocks=2, - axes_dims=[8, 4, 4], - visual_cond=False, - instruct_type="noise", - attention_params={"512": {"type": "flash"}}, - use_text=False, -) -TINY_PIFLOW = dict( - nfe=2, - num_policy_substeps=8, - final_step_size_scale=0.5, - shift=5.0, - n_grid=3, - eps=1e-6, -) -SR_PARAMS = dict( - scale_factor={"512": [1.0, 1.0, 1.0]}, - visual_size=[512], - scheduler_scale=5.0, - lq_noise_scale=0.7, - lq_noise_type="ddpm", - lq_channel_noise_scale=0.0, - cap_noise_timestep=False, - fps=24, -) - -def flat_config(**changes): - """A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied.""" - flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3) - flat.update( - piflow_nfe=2, - piflow_num_policy_substeps=8, - piflow_final_step_size_scale=0.5, - piflow_shift=5.0, - piflow_eps=1e-6, - attribute_overrides={"instruct_type": "noise"}, - sr_visual_size=[512], - sr_scale_factor={"512": [1.0, 1.0, 1.0]}, - sr_scheduler_scale=5.0, - sr_lq_noise_scale=0.7, - sr_lq_noise_type="ddpm", - sr_lq_channel_noise_scale=0.0, - sr_cap_noise_timestep=False, - sr_fps=24, - ) - flat.update(changes) - return flat - - -def official_config(**changes): - """An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head - width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys. - """ - official = { - key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",) - } - official.update( - out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"], - attribute_overrides=None, - sr_params=dict(SR_PARAMS), - ) - official.update(changes) - return official - - -def parse(flat): +def parse(**changes): config = Kandinsky6SRDitConfig() - config.update_model_arch(flat) + config.update_model_arch(dict(TINY_DIT, **changes)) return config.arch_config @@ -111,8 +37,7 @@ def parse(flat): # DiT arch config # --------------------------------------------------------------------------- # def test_arch_config_parses_an_official_config_and_maps_checkpoint_names(): - arch = parse(official_config()) - assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler + arch = parse() assert arch.out_visual_dim == 12 and arch.attribute_overrides == {} assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4) assert (arch.hidden_size, arch.num_attention_heads) == (32, 2) @@ -146,7 +71,7 @@ def test_arch_config_parses_an_official_config_and_maps_checkpoint_names(): def test_unknown_sr_params_keys_are_rejected(): with pytest.raises(ValueError, match="sr_params has unsupported keys"): - parse(official_config(sr_params={"fps": 24, "new_knob": 1})) + parse(sr_params={"fps": 24, "new_knob": 1}) @pytest.mark.parametrize( @@ -159,6 +84,7 @@ def test_unknown_sr_params_keys_are_rejected(): ({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"), ({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"), ({"piflow_eps": None}, ValueError, "piflow_eps"), + ({"n_grid": 3}, ValueError, "n_grid"), ({"patch_size": [2, 2, 2]}, ValueError, "patch_size"), ({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"), ], @@ -171,25 +97,28 @@ def test_unsupported_or_unknown_config_fails_loudly_naming_the_key( keys raise ValueError (``update_model_arch`` alone would park unknown keys in ``extra_attrs``).""" with pytest.raises(error, match=flag): - parse(flat_config(**changes)) + parse(**changes) def test_sparse_attention_request_is_detected_from_attention_params(): - assert parse(flat_config()).requested_sparse_attention() is None - nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}}) - assert parse(nabla).requested_sparse_attention() == "nabla" - # an override can switch it off again - overridden = flat_config( + assert parse().requested_sparse_attention() is None + assert ( + parse( + attention_params={"512": {"type": "nabla", "P": 0.9}} + ).requested_sparse_attention() + == "nabla" + ) + overridden = parse( attention_params={"512": {"type": "nabla"}}, attribute_overrides={"attention_params": {"512": {"type": "flash"}}}, ) - assert parse(overridden).requested_sparse_attention() is None + assert overridden.requested_sparse_attention() is None def test_default_arch_config_is_valid_and_text_free(): """Registry / ``PipelineConfig.from_kwargs`` instantiate the default config.""" arch = Kandinsky6SRArchConfig() - assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1 + assert arch.use_text is False # --------------------------------------------------------------------------- # diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py index 8f413952ba..0eaa63b346 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py @@ -37,13 +37,13 @@ from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import ( _SpatialChunkedConv3d, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( - Kandinsky6SRDecodeStage, -) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( SR_DENOISED_KEY, SR_TILES_KEY, ) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import ( + Kandinsky6SRDecodeStage, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( decode_chunks, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py index 258ae0ee8e..83a6f876e1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py @@ -1,12 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Check SR prompt admission and request fields without running a GPU scheduler. +"""Prompt admission and SR fields through the real JSON and multipart endpoints.""" -JSON uses TestClient; multipart tests call the parsed endpoint directly. The -full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads. -""" - -import asyncio -import os from types import SimpleNamespace from unittest.mock import AsyncMock, patch @@ -22,269 +16,80 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( ) from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.entrypoints.openai import video_api -from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( resolve_sampling_params_cls, ) -from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import ( - _build_video_sampling_params, - _multipart_video_extras, -) - -_SR_EXTRA_FIELDS = frozenset( - { - "sr_resolution_scale", - "sr_tiles_batch_size", - "sr_tile_min_overlap", - "sr_target_resolution", - "sr_target_resize_mode", - } -) -def _fake_server_args(**overrides) -> SimpleNamespace: - base = dict( +@pytest.mark.parametrize("multipart", [False, True], ids=["json", "multipart"]) +@pytest.mark.parametrize("prompt_required", [False, True], ids=["sr", "generic"]) +def test_prompt_admission_and_sr_fields(tmp_path, multipart, prompt_required): + args = SimpleNamespace( pipeline_config=Kandinsky6SRPipelineConfig(), pipeline_class_name="Kandinsky6SRPipeline", backend="auto", model_id=None, - model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", + model_path="kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", served_model_name="kandinsky6-sr-test", - input_save_path=os.path.expanduser( - "~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads" - ), - output_path=os.path.expanduser( - "~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output" - ), + input_save_path=str(tmp_path / "uploads"), + output_path=str(tmp_path / "outputs"), attention_backend_config={}, enable_trace=False, comfyui_mode=False, num_gpus=1, ) - base.update(overrides) - return SimpleNamespace(**base) - - -def _patched_global_server_args(server_args): - return patch( - "sglang.multimodal_gen.runtime.server_args.server_args._global_server_args", - server_args, + assert resolve_sampling_params_cls(args) is Kandinsky6SRSamplingParams + params_cls = SamplingParams if prompt_required else Kandinsky6SRSamplingParams + fields = dict( + sr_resolution_scale=4, + sr_tiles_batch_size=3, + sr_tile_min_overlap=0.3, + sr_target_resolution="hd", + sr_target_resize_mode="fit", + ) + assert Kandinsky6SRSamplingParams.video_request_extra_fields() == fields.keys() + assert SamplingParams.video_request_extra_fields() == frozenset() + payload = dict(video_path="unused.mp4", **fields) + request = ( + {"files": {key: (None, str(value)) for key, value in payload.items()}} + if multipart + else {"json": payload} ) - - -# --------------------------------------------------------------------------- # -# The two declared pieces (SamplingParams-level) -# --------------------------------------------------------------------------- # -def test_prompt_optional_defaults_false_and_sr_overrides_it(): - """A generic pipeline keeps requiring ``prompt``; only the SR params opt out.""" - assert SamplingParams.prompt_optional is False - assert Kandinsky6SRSamplingParams.prompt_optional is True - - -@pytest.mark.parametrize( - "sampling_params_cls, status", - [(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)], -) -def test_json_prompt_requirement_matches_pipeline( - tmp_path, sampling_params_cls, status -): app = FastAPI() app.include_router(video_api.router) - server_args = _fake_server_args( - input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs") - ) dispatch = AsyncMock() with ( - _patched_global_server_args(server_args), - patch.object( - video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls + patch( + "sglang.multimodal_gen.runtime.server_args.server_args._global_server_args", + args, ), + patch.object(video_api, "resolve_sampling_params_cls", return_value=params_cls), patch.object(video_api, "_dispatch_job_async", dispatch), TestClient(app) as client, ): - response = client.post( - "/v1/videos", - json={"video_path": "unused.mp4", "sr_resolution_scale": 4}, - ) - assert response.status_code == status, response.text - if status == 200: + response = client.post("/v1/videos", **request) + assert response.status_code == (400 if prompt_required else 200), response.text + if prompt_required: + dispatch.assert_not_called() + else: assert response.json()["status"] == "queued" dispatch.assert_awaited_once() _, batch = dispatch.call_args.args - assert batch.prompt == "" - assert batch.sampling_params.sr_resolution_scale == 4 - else: - dispatch.assert_not_called() + assert isinstance(batch.sampling_params, Kandinsky6SRSamplingParams) + assert batch.prompt == "" and batch.video_path == "unused.mp4" + assert {key: vars(batch.sampling_params)[key] for key in fields} == fields -def test_sr_declares_its_cli_flags_as_multipart_extra_fields(): - assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS - # The base default (what every other model still gets) stays empty. - assert SamplingParams.video_request_extra_fields() == frozenset() - - -# --------------------------------------------------------------------------- # -# The real multipart-field allowlist (video_api._multipart_video_extras) -# --------------------------------------------------------------------------- # -def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class(): - raw_form = { - "video_path": "unused.mp4", - "sr_resolution_scale": "4", - "sr_tiles_batch_size": "3", - "not_a_declared_field": "dropped", - } - sr_extras = _multipart_video_extras( - raw_form, +@pytest.mark.parametrize("params_cls", [Kandinsky6SRSamplingParams, SamplingParams]) +def test_multipart_allowlist_drops_undeclared_fields(params_cls): + extras = video_api._multipart_video_extras( + {"sr_resolution_scale": "4", "sr_tiles_batch_size": "3", "undeclared": "1"}, extra_body=None, extra_params=None, - sampling_params_cls=Kandinsky6SRSamplingParams, + sampling_params_cls=params_cls, ) - assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4" - assert sr_extras["sr_tiles_batch_size"] == 3 - assert "not_a_declared_field" not in sr_extras - - # A model without the override (the pre-fix behaviour) drops the sr_* fields too. - base_extras = _multipart_video_extras( - raw_form, - extra_body=None, - extra_params=None, - sampling_params_cls=SamplingParams, + assert extras == ( + {"sr_resolution_scale": 4, "sr_tiles_batch_size": 3} + if params_cls is Kandinsky6SRSamplingParams + else {} ) - assert "sr_resolution_scale" not in base_extras - assert "sr_tiles_batch_size" not in base_extras - - -# --------------------------------------------------------------------------- # -# The registry-backed class resolution the admission gate reads -# --------------------------------------------------------------------------- # -def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline(): - server_args = _fake_server_args() - assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams - - -# --------------------------------------------------------------------------- # -# End-to-end request parsing: a prompt-less request field set -> sampling params -# --------------------------------------------------------------------------- # -def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields(): - """The real kwargs-building code (``_build_video_sampling_params``), given a - request built the way the multipart branch builds one for a prompt-optional - pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a - real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error.""" - from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( - VideoGenerationsRequest, - ) - - server_args = _fake_server_args() - raw_form = { - "video_path": "unused.mp4", - "sr_resolution_scale": "4", - "sr_tiles_batch_size": "3", - } - extras = _multipart_video_extras( - raw_form, - extra_body=None, - extra_params=None, - sampling_params_cls=Kandinsky6SRSamplingParams, - ) - req = VideoGenerationsRequest( - prompt="", # what video_api.py substitutes for a missing prompt - video_path=extras.pop("video_path"), - **extras, - ) - - with _patched_global_server_args(server_args): - sampling_params = _build_video_sampling_params("gap1-request-id", req) - - assert isinstance(sampling_params, Kandinsky6SRSamplingParams) - assert sampling_params.prompt == "" - assert sampling_params.sr_resolution_scale == 4 - assert sampling_params.sr_tiles_batch_size == 3 - - -# --------------------------------------------------------------------------- # -# The full endpoint, called directly (see module docstring for why not TestClient) -# --------------------------------------------------------------------------- # -class _FakeMultipartRequest: - """Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly.""" - - def __init__(self, form: dict): - self.headers = {"content-type": "multipart/form-data; boundary=x"} - self._form = form - - async def form(self): - return self._form - - -def test_multipart_sr_request_without_prompt_is_accepted(): - """A multipart SR request with no ``prompt`` field must be admitted (no 400), the - ``sr_*`` fields must reach the queued request, and it must fail on the pre-fix - code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would - reject this before any of the rest of ``create_video`` ran.""" - server_args = _fake_server_args() - request = _FakeMultipartRequest( - {"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"} - ) - dispatched: list[tuple] = [] - - async def _fake_dispatch(job_id, batch, **kwargs): - dispatched.append((job_id, batch, kwargs)) - - async def run(): - with ( - _patched_global_server_args(server_args), - patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch), - ): - response = await video_api.create_video( - request, - prompt=None, # <-- the field under test: no prompt supplied - enhance_prompt=None, - task_type=None, - x264_preset=None, - perf_dump_path=None, - input_reference=None, - reference_url=None, - video_reference=None, - video_url=None, - video_path="unused.mp4", - model=None, - n=1, - num_outputs_per_prompt=None, - seconds=None, - size=None, - fps=None, - num_frames=None, - seed=None, - generator_device="cuda", - negative_prompt=None, - guidance_scale=None, - guidance_scale_2=None, - true_cfg_scale=None, - num_inference_steps=None, - max_sequence_length=None, - flow_shift=None, - enable_teacache=None, - enable_frame_interpolation=None, - frame_interpolation_exp=None, - frame_interpolation_scale=None, - frame_interpolation_model_path=None, - enable_upscaling=None, - upscaling_model_path=None, - upscaling_scale=None, - output_quality=None, - output_compression=None, - output_path=None, - extra_params=None, - extra_body=None, - ) - await asyncio.sleep(0) - return response - - response = asyncio.run(run()) - - assert isinstance(response, VideoResponse) - assert response.status == "queued" - assert len(dispatched) == 1 - _, batch, _ = dispatched[0] - assert batch.prompt == "" - assert batch.sampling_params.sr_resolution_scale == 4 - assert batch.sampling_params.sr_tiles_batch_size == 3 diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py index 572ef58871..dc600190f4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py @@ -3,11 +3,10 @@ import copy import json -import os import pytest import torch -from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL +from kandinsky6_sr_tiny_components import TINY_DIT, TINY_KVAE, TINY_LU_MODEL from safetensors.torch import load_file, save_file from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( @@ -19,10 +18,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( Kandinsky6SRPipelineConfig, ) -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) from sglang.multimodal_gen.runtime.loader.component_loaders import ( component_loader as component_loader_module, ) @@ -34,6 +29,7 @@ from sglang.multimodal_gen.runtime.loader.fsdp_load import ( load_model_from_full_model_state_dict, ) from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( Kandinsky6SRTransformer3DModel, ) @@ -42,39 +38,7 @@ from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscale ) from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE -TINY_DIT = dict( - in_visual_dim=4, - in_text_dim=8, - in_text_dim2=8, - time_dim=16, - out_visual_dim=4, - patch_size=[1, 2, 2], - model_dim=32, - ff_dim=64, - num_text_blocks=0, - num_visual_blocks=2, - axes_dims=[8, 4, 4], - visual_cond=False, - instruct_type="noise", - use_text=False, - n_grid=3, - attribute_overrides={"instruct_type": "noise"}, -) - - -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29508", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +pytestmark = pytest.mark.usefixtures("single_process_model_parallel") @pytest.fixture(autouse=True) @@ -138,100 +102,48 @@ def _load_through_the_real_loader(checkpoint, **changes): return model -# --------------------------------------------------------------------------- # -# DiT -# --------------------------------------------------------------------------- # def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt(): - """Positive control for the failure tests below, and the meta-device path: RoPE - tables and time-embedding frequencies are not in the checkpoint and must be rebuilt - by ``post_load_weights`` so that a loaded model actually runs.""" + """Meta-init must rebuild non-checkpoint RoPE and time-embedding buffers.""" reference = _dit() checkpoint = _official_state_dict(reference) model = _load_through_the_real_loader(checkpoint) model.post_load_weights() assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()]) - assert not model.time_embeddings.freqs.is_meta torch.testing.assert_close(model.pooled_bias, reference.pooled_bias) + assert model.out_layer.out_layer.weight.shape[0] == 4 * TINY_DIT["out_visual_dim"] device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) x = torch.randn(1, 2, 4, 4, 4, device=device) rope_pos = [torch.arange(2, device=device) for _ in range(3)] - out = model(x, torch.tensor([500.0], device=device), rope_pos) + with set_forward_context(current_timestep=0, attn_metadata=None): + out = model(x, torch.tensor([500.0], device=device), rope_pos) assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all() @pytest.mark.parametrize( - "missing", + "key,unexpected,message", [ - "pooled_bias", - "visual_transformer_blocks.0.visual_modulation.out_layer.bias", + ("pooled_bias", False, "was not loaded"), + ( + "visual_transformer_blocks.0.visual_modulation.out_layer.bias", + False, + "was not loaded", + ), + ("text_embeddings.in_layer.weight", True, "unexpected keys"), ], ) -def test_missing_checkpoint_tensor_fails_the_load(missing): - """The generic loader zero-fills missing parameters whose names contain ``bias`` (and - would only warn about others). For this model every parameter is required, so a - partial checkpoint must raise instead of yielding a silently different network.""" +def test_incomplete_or_unexpected_checkpoint_fails_the_load(key, unexpected, message): + # even bias parameters must fail instead of the generic loader's zero-fill fallback checkpoint = _official_state_dict(_dit()) - del checkpoint[missing] - with pytest.raises(ValueError, match="was not loaded"): + if unexpected: + checkpoint[key] = torch.zeros(2, 2) + else: + del checkpoint[key] + with pytest.raises(ValueError, match=message): _load_through_the_real_loader(checkpoint) -def test_unexpected_checkpoint_tensor_fails_the_load(): - checkpoint = _official_state_dict(_dit()) - checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2) - with pytest.raises(ValueError, match="unexpected keys"): - _load_through_the_real_loader(checkpoint) - - -def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys(): - """An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives - in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``. - """ - official = { - key: value - for key, value in TINY_DIT.items() - if key not in ("n_grid", "attribute_overrides") - } - official.update( - out_visual_dim=12, - attribute_overrides=None, - sr_params=dict( - scale_factor={"512": [1.0, 1.0, 1.0]}, - visual_size=[512], - scheduler_scale=5.0, - lq_noise_scale=0.7, - lq_noise_type="ddpm", - lq_channel_noise_scale=0.0, - cap_noise_timestep=False, - fps=24, - ), - ) - config = Kandinsky6SRDitConfig() - config.update_model_arch(official) - with torch.device("meta"): - model = Kandinsky6SRTransformer3DModel(config, official) - # head width = prod(patch_size) * the TOTAL out_visual_dim - assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12 - assert model.base_out_visual_dim == 4 and model.n_grid == 1 - - load_model_from_full_model_state_dict( - model, - model.preprocess_loaded_state_dict( - iter( - _official_state_dict(_dit()).items() - ) # same shapes as the legacy DX head - ), - torch.device("cpu"), - torch.float32, - strict=False, - param_names_mapping=get_param_names_mapping(model.param_names_mapping), - ) - model.post_load_weights() - assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()]) - - def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise(): """A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise`` override would feed ``in`` channels. That mismatch must be reported, and the @@ -260,9 +172,6 @@ def test_rope_angle_tables_stay_fp32_when_the_module_is_cast(): assert buffer.dtype == torch.float32, name -# --------------------------------------------------------------------------- # -# Fake server args for the component loaders -# --------------------------------------------------------------------------- # class FakeServerArgs: def __init__(self): self.pipeline_config = Kandinsky6SRPipelineConfig() @@ -365,12 +274,21 @@ def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True): return directory -def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path): - directory = _tiny_lu_bundle(tmp_path) - bank, _ = LatentUpscalerLoader().load( - str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" +@pytest.mark.parametrize("legacy,library", [(False, "diffusers"), (True, "kandinsky6")]) +def test_latent_upscaler_loading_and_default_scale_order(tmp_path, legacy, library): + models = [ + {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} + for scale in (("4x", "2x") if legacy else ("2x", "4x")) + ] + directory = _tiny_lu_bundle(tmp_path, models=models, include_scales=not legacy) + loader = component_loader_module.ComponentLoader.for_component_type( + "latent_upscaler", library ) + assert isinstance(loader, LatentUpscalerLoader) + bank, _ = loader.load(str(directory), FakeServerArgs(), "latent_upscaler", library) assert bank.scales == (2, 4) + assert bank._models[0] is bank.for_scale(2) + assert bank._models[1] is bank.for_scale(4) assert not bank.training assert all( p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters() @@ -382,75 +300,35 @@ def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path): bank.upscale(z, scale=8) -def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path): - models = [ - {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} - for scale in ("4x", "2x") - ] - directory = _tiny_lu_bundle( - tmp_path, - models=models, - include_scales=False, - ) - bank, _ = LatentUpscalerLoader().load( - str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" - ) - assert bank.scales == (2, 4) - assert bank._models[0] is bank.for_scale(2) - assert bank._models[1] is bank.for_scale(4) - - -def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library(): - """The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...].""" - for library in ("diffusers", "kandinsky6"): - loader = component_loader_module.ComponentLoader.for_component_type( - "latent_upscaler", library - ) - assert isinstance(loader, LatentUpscalerLoader) +def test_latent_upscaler_rejects_the_wrong_library(): with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"): component_loader_module.ComponentLoader.for_component_type( "latent_upscaler", "transformers" ) -def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path): - """``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the - customized loader raises, which hides real errors. This component must re-raise, - with the offending config key or tensor named.""" +@pytest.mark.parametrize("error_kind", ["Missing key", "bogus_field"]) +def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path, error_kind): directory = _tiny_lu_bundle(tmp_path) - weights_path = directory / "diffusion_pytorch_model.safetensors" - weights = load_file(str(weights_path)) - weights.pop(next(iter(weights))) - save_file(weights, str(weights_path)) + if error_kind == "Missing key": + path = directory / "diffusion_pytorch_model.safetensors" + weights = load_file(str(path)) + weights.pop(next(iter(weights))) + save_file(weights, str(path)) + else: + path = directory / "config.json" + config = json.loads(path.read_text()) + config["models"][0]["model"]["bogus_field"] = 1 + path.write_text(json.dumps(config)) with pytest.raises(RuntimeError, match="native fallback is disabled") as error: LatentUpscalerLoader().load( str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" ) - assert "Missing key" in str(error.value.__cause__) - - bad = copy.deepcopy(TINY_LU_MODEL) - bad["bogus_field"] = 1 - directory = _tiny_lu_bundle( - tmp_path / "bad", - models=[{"target_scale": "2x", "model": TINY_LU_MODEL}], - scales=(2,), - ) - config_path = directory / "config.json" - config = json.loads(config_path.read_text()) - config["models"][0]["model"] = bad - config_path.write_text(json.dumps(config)) - with pytest.raises(RuntimeError, match="native fallback is disabled") as error: - LatentUpscalerLoader().load( - str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" - ) - assert "bogus_field" in str(error.value.__cause__) + assert error_kind in str(error.value.__cause__) def test_motion_attention_is_not_a_supported_entry_key(): - """The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` / - commit 950a5edb) only implements the released checkpoint, which never trains - natten/shifted-window motion attention: a config that asks for it must fail loudly - at construction instead of silently building an unsupported backend.""" + """Unsupported motion attention must fail instead of silently using dense attention.""" with_motion = { **copy.deepcopy(TINY_LU_MODEL), "enable_x2_entry": False, diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py index 75cff0afff..4c49f8f583 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py @@ -3,7 +3,6 @@ import copy import json -import os from collections import Counter from types import SimpleNamespace from typing import NamedTuple @@ -32,10 +31,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( Kandinsky6SRPipelineConfig, ) from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( PipelineComponentLoader, ) @@ -61,9 +56,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import ( from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( build_sampling_spec, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( - PiflowParams, -) class Bundle(NamedTuple): @@ -77,7 +69,6 @@ class Bundle(NamedTuple): head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid`` scheduler_scale: float # ``sr_params.scheduler_scale`` - calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5 FLOW = Bundle( @@ -89,7 +80,6 @@ FLOW = Bundle( head_width=64, n_grid=1, scheduler_scale=5.0, - calls_per_tile=5, ) DISTILLED = Bundle( model_index=DISTILLED_MODEL_INDEX, @@ -100,26 +90,13 @@ DISTILLED = Bundle( head_width=640, n_grid=10, scheduler_scale=3.5, - calls_per_tile=2, ) both_repos = pytest.mark.parametrize( "bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"] ) -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29509", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +pytestmark = pytest.mark.usefixtures("single_process_model_parallel") def _prefixes(keys, depth=2): @@ -162,17 +139,18 @@ def test_real_transformer_config_builds_the_release_dit(bundle): # ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its # n_grid in the scheduler config assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792) - assert model.base_out_visual_dim == 64 and model.n_grid == 1 - assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width + assert bundle.n_grid * model.in_visual_dim == bundle.head_width assert model.instruct_type == "hybrid_anchor" and model.visual_cond assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input arch = config.arch_config assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]} - assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow + assert arch.sr_scheduler_scale == bundle.scheduler_scale @both_repos -def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path): +def test_loaded_scheduler_controls_sampling_and_rejects_a_mismatched_head( + bundle, tmp_path +): scheduler = _load_scheduler(bundle, tmp_path) # PiflowScheduler subclasses the flow-matching scheduler: compare exact types assert type(scheduler) is bundle.scheduler_cls @@ -180,43 +158,48 @@ def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_pa if bundle is DISTILLED: assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10) - -@both_repos -def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path): - """The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever - ``num_inference_steps`` is), the flow-matching repo to flow-Euler - (``num_inference_steps`` calls per tile directly -- this repo's own convention, not - the upstream Diffusers pipeline's timestep-grid-point count). - """ - scheduler = _load_scheduler(bundle, tmp_path) arch = _dit_config(bundle).arch_config + for num_steps in (3, 5, 9): + spec = _sampling_spec(arch, scheduler, num_steps) + assert spec.is_piflow == (bundle is DISTILLED) + assert spec.num_steps == (2 if bundle is DISTILLED else num_steps) + mismatched = _dit_config(FLOW if bundle is DISTILLED else DISTILLED).arch_config + with pytest.raises(ValueError, match="head.*same checkpoint"): + _sampling_spec(mismatched, scheduler) - def spec(num_steps): - return build_sampling_spec( - arch=arch, - tiling_scale=2, - seed=42, - num_steps=num_steps, - tiles_batch_size=1, - tile_min_overlap=0.2, - scheduler=scheduler, + +def _sampling_spec(arch, scheduler, num_steps=5): + return build_sampling_spec( + arch=arch, + scheduler=scheduler, + tiling_scale=2, + seed=42, + num_steps=num_steps, + tiles_batch_size=1, + tile_min_overlap=0.2, + ) + + +@pytest.mark.parametrize("nfe", [None, 0, -1]) +def test_piflow_requires_a_positive_checkpoint_step_count(nfe): + with pytest.raises(ValueError, match="nfe >= 1"): + _sampling_spec( + _dit_config(DISTILLED).arch_config, PiflowScheduler(nfe=nfe, n_grid=10) ) - assert spec(5).steps_per_chunk == bundle.calls_per_tile - assert spec(5).scheduler_scale == bundle.scheduler_scale - if bundle is DISTILLED: - assert spec(5).piflow == PiflowParams( - nfe=2, - num_policy_substeps=128, - final_step_size_scale=0.5, - shift=3.5, - n_grid=10, - eps=1e-6, - ) - assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2 - else: - assert spec(5).piflow is None - assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9) + +def test_sampling_rejects_missing_scheduler_steps_and_unsupported_cap(): + arch = _dit_config(DISTILLED).arch_config + for scheduler in (None, object()): + with pytest.raises(ValueError, match="scheduler|Scheduler"): + _sampling_spec(arch, scheduler) + scheduler = PiflowScheduler(nfe=2, n_grid=10) + with pytest.raises(ValueError, match="num_inference_steps"): + _sampling_spec(arch, scheduler, num_steps=0) + arch.sr_cap_noise_timestep = True + arch.attribute_overrides = {"instruct_type": "noise"} + with pytest.raises(NotImplementedError, match="cap_noise_timestep"): + _sampling_spec(arch, scheduler) def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys(): diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py index 76e3244c12..abf617fa70 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py @@ -23,7 +23,12 @@ from types import SimpleNamespace import pytest import torch -from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve +from kandinsky6_sr_tiny_components import ( + TINY_KVAE, + TINY_LU_MODEL, + TINY_PIFLOW, + super_resolve, +) from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( Kandinsky6SRDitConfig, @@ -31,14 +36,11 @@ from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( Kandinsky6SRVAEConfig, ) -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) from sglang.multimodal_gen.runtime.loader.utils import ( get_param_names_mapping, hf_to_custom_state_dict, ) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( Kandinsky6SRTransformer3DModel, ) @@ -58,12 +60,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( build_dit_spec, build_sampling_spec, - effective_scheduler, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( - denoise_with_scheduler, - euler_start_timestep, - make_dit_fn, + SamplingSpec, + denoise_chunks, module_dtype, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( @@ -105,14 +105,6 @@ TINY_DIT_CFG = dict( attention_params={"512": {"type": "flash"}}, use_text=False, ) -TINY_PIFLOW = dict( - nfe=2, - num_policy_substeps=8, - final_step_size_scale=0.5, - shift=5.0, - n_grid=3, - eps=1e-6, -) TINY_SR_PARAMS = dict( scale_factor={512: [1.0, 1.0, 1.0]}, visual_size=[512], @@ -203,20 +195,7 @@ from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402 ) from omegaconf import OmegaConf # noqa: E402 - -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29507", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +pytestmark = pytest.mark.usefixtures("single_process_model_parallel") def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None: @@ -240,40 +219,16 @@ def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None return dit -def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict: - """The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields).""" - flat = dict(cfg) - flat["patch_size"] = list(flat["patch_size"]) - flat["axes_dims"] = list(flat["axes_dims"]) - flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1 - if piflow is not None: - flat.update( - piflow_nfe=piflow["nfe"], - piflow_num_policy_substeps=piflow["num_policy_substeps"], - piflow_final_step_size_scale=piflow["final_step_size_scale"], - piflow_shift=piflow["shift"], - piflow_eps=piflow["eps"], - ) - flat["attribute_overrides"] = dict(overrides) - flat.update( - sr_visual_size=[512], - sr_scale_factor={"512": [1.0, 1.0, 1.0]}, - sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"], - sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"], - sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"], - sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"], - sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"], - sr_fps=TINY_SR_PARAMS["fps"], - ) - return flat - - def build_port_dit( reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict ) -> Kandinsky6SRTransformer3DModel: """Port DiT loaded with the reference weights (``model.`` prefix), strictly.""" - flat = flat_transformer_config( - cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides + cfg = dict(cfg or TINY_DIT_CFG) + flat = dict( + cfg, + out_visual_dim=cfg["out_visual_dim"] * (piflow["n_grid"] if piflow else 1), + attribute_overrides=dict(overrides), + sr_params=dict(TINY_SR_PARAMS), ) dit_config = Kandinsky6SRDitConfig() dit_config.update_model_arch(flat) @@ -324,13 +279,14 @@ def _port_forward(model, x, time, *, scale_factor, motion_score=None): torch.arange(height // model.patch_size[1]), torch.arange(width // model.patch_size[2]), ] - return model( - x, - time, - rope_pos, - scale_factor=scale_factor, - motion_score=motion_score, - ) + with set_forward_context(current_timestep=0, attn_metadata=None): + return model( + x, + time, + rope_pos, + scale_factor=scale_factor, + motion_score=motion_score, + ) def _random_latent( @@ -427,28 +383,36 @@ def _reference_loop_inputs(x): return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long) -def _port_dit_fn(model, x, scale_factor, use_motion_score=False): - return make_dit_fn( - model, - latent_frames_hw=tuple(x.shape[1:4]), - patch_size=model.patch_size, - scale_factor=scale_factor, - use_motion_score=use_motion_score, +def _port_sample(model, x, scale, scheduler, steps, capped=False): + spec = SamplingSpec( + tiling_scale=2, + tiles_batch_size=1, + seed=42, + num_steps=steps, + is_piflow=isinstance(scheduler, PiflowScheduler), + tile_min_overlap=0.2, + visual_size=512, + scale_factor=scale, + lq_noise_scale=0.7, + lq_noise_type="ddpm", + lq_channel_noise_scale=0.0, + cap_noise_timestep=capped, ) + return denoise_chunks( + [x.clone()], + model, + scheduler, + dit_spec=build_dit_spec(model), + spec=spec, + device=x.device, + step_context=lambda step: set_forward_context( + current_timestep=step, attn_metadata=None + ), + )[0] def test_piflow_loop_matches_reference_piflow_generate(): - """Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles. - - The reference packs the tiles along time with one per-frame sigma; the port batches - ``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU. - - The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule`` - / ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through - ``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the - ``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same - per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did. - """ + """Compare the packed reference loop with the batched native PiflowScheduler.""" from kandinsky_sr.core.algo.piflow_sampler import piflow_generate reference = build_reference_dit(piflow=TINY_PIFLOW) @@ -460,7 +424,6 @@ def test_piflow_loop_matches_reference_piflow_generate(): visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) packed = x.reshape(-1, *x.shape[2:]).clone() scheduler = PiflowScheduler(**TINY_PIFLOW) - scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu") with torch.no_grad(): expected = piflow_generate( packed, @@ -476,13 +439,7 @@ def test_piflow_loop_matches_reference_piflow_generate(): device="cpu", **TINY_PIFLOW, ) - actual = denoise_with_scheduler( - x.clone(), - _port_dit_fn(model, x, scale), - scheduler, - channels=4, - is_piflow=True, - ) + actual = _port_sample(model, x, scale, scheduler, TINY_PIFLOW["nfe"]) torch.testing.assert_close( actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 ) @@ -492,18 +449,8 @@ def test_piflow_loop_matches_reference_piflow_generate(): "capped", [False, True], ids=["full_range", "capped_noise_start"] ) def test_euler_loop_matches_reference_generate(capped): - """Guards the warped timesteps, per-step deltas and (for a wide input) the in-place - update of the first channels only, incl. the ``cap_noise_timestep`` start time. - - The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4 - Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent - port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace - array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift`` - warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself -- - algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4 - step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for- - bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array. - """ + """Compare noise caps and channel updates with the reference's five-point grid + (four DiT calls); both schedulers append the same terminal zero.""" from kandinsky_sr.core.algo.utils import generate cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor") @@ -517,16 +464,11 @@ def test_euler_loop_matches_reference_generate(capped): reference.instruct_type, reference.visual_cond = "noise", True x = _random_latent(2 * 4 + 1, seed=9) scale = (1.0, 1.5, 1.5) - start = euler_start_timestep( - cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise" - ) - assert start == (0.7 if capped else 1.0) + start = 0.7 if capped else 1.0 visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) packed = x.reshape(-1, *x.shape[2:]).clone() num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0) - sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist() - scheduler.set_timesteps(sigmas=sigmas, device="cpu") with torch.no_grad(): expected = generate( packed, @@ -546,13 +488,7 @@ def test_euler_loop_matches_reference_generate(capped): 5.0, start_timestep=start, ) - actual = denoise_with_scheduler( - x.clone(), - _port_dit_fn(model, x, scale, use_motion_score=True), - scheduler, - channels=4, - is_piflow=False, - ) + actual = _port_sample(model, x, scale, scheduler, num_inference_steps, capped) torch.testing.assert_close( actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 ) @@ -663,6 +599,9 @@ def build_stacks(*, piflow, use_lu): vae.load_state_dict(dict(ref_vae.state_dict()), strict=True) port = { "vae": vae.eval(), + "scheduler": PiflowScheduler(**piflow) + if piflow + else FlowMatchEulerDiscreteScheduler(shift=TINY_SR_PARAMS["scheduler_scale"]), "dit": build_port_dit( ref_dit, piflow=piflow, overrides={"instruct_type": "noise"} ), @@ -728,7 +667,10 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps): if pre != 1.0: video = tiling.pre_upscale_video(video, pre, 16) arch = port["dit"].config - port_num_steps = num_steps if arch.is_piflow else num_steps - 1 + scheduler = port["scheduler"] + port_num_steps = ( + num_steps if isinstance(scheduler, PiflowScheduler) else num_steps - 1 + ) spec = build_sampling_spec( arch=arch, tiling_scale=tiling_scale, @@ -736,6 +678,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps): num_steps=port_num_steps, tiles_batch_size=tiles_batch_size, tile_min_overlap=0.2, + scheduler=scheduler, ) bank = port["bank"] use_lu = bank is not None and bank.for_scale(tiling_scale) is not None @@ -746,7 +689,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps): dit=port["dit"], dit_spec=build_dit_spec(port["dit"]), spec=spec, - scheduler=effective_scheduler(spec, None), + scheduler=scheduler, device="cpu", upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None, lu_dtype=module_dtype(bank) if use_lu else None, @@ -782,19 +725,10 @@ def _assert_uint8_close(actual, expected, *, max_diff: int = 1): def test_tiled_piflow_super_resolution_matches_reference( use_lu, scale, hw, tiles_batch_size ): - """Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or - tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with - truncation to uint8 and Hann stitching, against ``run_tiled_sr`` / - ``run_tiled_sr_from_pixels`` with identical random weights. + """Compare tiled pixel/LU paths with identical random weights and seeds. - ``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the - training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to - 0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``). - The port now deliberately matches the real ``PiflowScheduler._policy_step`` - convention (``raw_dst=eps``) instead, which real checkpoints always run through. - That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up - here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing - rounding-boundary noise (still >99.99% exact pixels), not as a structural bug. + The reference ends at raw_dst=0, whereas PiflowScheduler ends at eps (1e-6). + Allow two uint8 levels of error while requiring more than 99.9% exact pixels. """ reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu) video = _random_video(9, *hw) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py deleted file mode 100644 index bf138031f0..0000000000 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py +++ /dev/null @@ -1,253 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Validate scheduler selection, DiT call counts and incompatible head rejection.""" - -import os -from typing import NamedTuple - -import pytest -from kandinsky6_sr_release_configs import ( - DISTILLED_SCHEDULER_CONFIG, - DISTILLED_TRANSFORMER_CONFIG, - FLOW_SCHEDULER_CONFIG, - FLOW_TRANSFORMER_CONFIG, -) -from kandinsky6_sr_tiny_components import ( - TINY_DIT, - build_components, - make_request, - make_stage, - random_video, -) - -from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( - Kandinsky6SRDitConfig, -) -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, -) -from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( - PiflowScheduler, -) -from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( - Kandinsky6SRDenoisingStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( - Kandinsky6SRLatentPrepStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - SR_VIDEO_KEY, - build_sampling_spec, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( - RESOLUTIONS, -) - -LATENT_WIDTH = TINY_DIT["in_visual_dim"] -N_GRID = ( - 3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields -) -NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles - - -def official_dit(head_width): - """Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL - head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config.""" - arch = { - key: value - for key, value in TINY_DIT.items() - if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_")) - } - return dict( - arch, - out_visual_dim=head_width, - sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]), - ) - - -FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head -DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head - - -def flow_scheduler(): - return FlowMatchEulerDiscreteScheduler(shift=5.0) - - -def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID): - return PiflowScheduler( - nfe=nfe, - n_grid=n_grid, - shift=5.0, - eps=1e-6, - final_step_size_scale=0.5, - num_policy_substeps=8, - ) - - -def release_scheduler(scheduler_cls, config): - return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")}) - - -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29510", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) - - -@pytest.fixture(autouse=True) -def tiny_resolutions(monkeypatch): - monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)]) - - -class SamplingRun(NamedTuple): - tiles: int - dit_calls: int - progress_totals: list - - -def run_denoising(dit_arch, scheduler, *, num_steps=5): - """Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count - the DiT calls next to the totals the denoising progress bar was opened with.""" - vae, dit, _, server_args = build_components((), dit_arch) - dit_calls = [] - dit.register_forward_hook(lambda module, args, output: dit_calls.append(1)) - latent_prep = make_stage( - Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler - ) - denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler) - progress_totals = [] - progress_bar = denoise.progress_bar - - def spy_on_progress_bar(**kwargs): - progress_totals.append(kwargs["total"]) - return progress_bar(**kwargs) - - denoise.progress_bar = spy_on_progress_bar - - batch = make_request(random_video(9, 64, 96), num_steps=num_steps) - batch = latent_prep.forward(batch, server_args) - tiles = len(batch.extra["kandinsky6_sr_chunks"]) - denoise.forward(batch, server_args) - return SamplingRun(tiles, len(dit_calls), progress_totals) - - -def sampling_spec(dit_arch, scheduler): - config = Kandinsky6SRDitConfig() - config.update_model_arch(dict(dit_arch)) - return build_sampling_spec( - arch=config.arch_config, - tiling_scale=2, - seed=42, - num_steps=5, - tiles_batch_size=1, - tile_min_overlap=0.2, - scheduler=scheduler, - ) - - -# --------------------------------------------------------------------------- # -# DiT calls per tile -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)]) -def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps( - nfe, num_steps -): - run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps) - assert run.tiles == NUM_TILES - assert run.dit_calls == NUM_TILES * nfe - assert run.progress_totals == [NUM_TILES * nfe] - - -@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)]) -def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile( - num_steps, calls_per_tile -): - run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps) - assert run.tiles == NUM_TILES - assert run.dit_calls == NUM_TILES * calls_per_tile - assert run.progress_totals == [NUM_TILES * calls_per_tile] - - -@pytest.mark.parametrize("num_steps", [3, 9]) -def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps): - """A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the - denoising stage synthesizes a matching PiflowScheduler from those arch fields - (run_spec.effective_scheduler) instead of leaving nothing to step.""" - run = run_denoising(TINY_DIT, None, num_steps=num_steps) - assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"] - assert run.progress_totals == [run.dit_calls] - - -# --------------------------------------------------------------------------- # -# Samplers that do not fit the DiT head -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "scheduler", - [flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None], - ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"], -) -def test_dx_head_never_runs_flow_euler(scheduler): - with pytest.raises(ValueError, match="flow-Euler"): - sampling_spec(DX_DIT, scheduler) - - -@pytest.mark.parametrize( - "dit_arch,n_grid,needed", - [ - (FLOW_DIT, N_GRID, "3 x 4 = 12"), - (DX_DIT, 2, "2 x 4 = 8"), - (DX_DIT, 4, "4 x 4 = 16"), - ], - ids=["single_head", "too_few_grids", "too_many_grids"], -) -def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed): - with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"): - sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid)) - - -@pytest.mark.parametrize("nfe", [0, -1]) -def test_pi_flow_needs_at_least_one_call_per_tile(nfe): - with pytest.raises(ValueError, match="nfe >= 1"): - sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe)) - with pytest.raises(ValueError, match="nfe >= 1"): - sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None) - - -def test_release_transformer_and_scheduler_of_different_repos_are_rejected(): - """The real configs of the two official repos: their transformers differ only in - the head width, so one repo's transformer with the other's scheduler must fail.""" - flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG) - distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG) - - assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5 - assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2 - with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"): - sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow) - with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"): - sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled) - - -def test_a_mismatched_bundle_fails_before_any_tile_is_processed(): - vae, dit, _, server_args = build_components((), DX_DIT) - dit_calls = [] - dit.register_forward_hook(lambda module, args, output: dit_calls.append(1)) - latent_prep = make_stage( - Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler() - ) - batch = make_request(random_video(9, 64, 96)) - - with pytest.raises(ValueError, match="flow-Euler"): - latent_prep.forward(batch, server_args) - assert dit_calls == [] - assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py index 7e19ff0625..1dd48ff8ad 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py @@ -2,14 +2,15 @@ """Native SR stage-chain parity, component residency and video/audio hand-offs.""" import copy -import os from functools import partial -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest import torch from kandinsky6_sr_tiny_components import ( + TINY_DIT, TINY_LU_MODEL, + TINY_PIFLOW, build_components, make_request, make_stage, @@ -21,9 +22,11 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( Kandinsky6SRSamplingParams, ) from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType -from sglang.multimodal_gen.runtime.distributed.parallel_state import ( - maybe_init_distributed_environment_and_model_parallel, - model_parallel_is_initialized, +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, ) from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( Kandinsky6SRLatentUpscalerBank, @@ -32,26 +35,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import ( Kandinsky6SRPipeline, ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( - Kandinsky6SRDecodeStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( - Kandinsky6SRDenoisingStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import ( - Kandinsky6SREncodeStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( - WARMUP_CLIP_FRAMES, - WARMUP_CLIP_HW, - Kandinsky6SRInputStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( - Kandinsky6SRLatentPrepStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import ( - Kandinsky6SROutputStage, -) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( SR_LR_LATENT_KEY, SR_PLAN_KEY, @@ -60,11 +43,20 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k SR_VIDEO_KEY, build_dit_spec, build_sampling_spec, - effective_scheduler, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( module_dtype, ) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import ( + WARMUP_CLIP_FRAMES, + WARMUP_CLIP_HW, + Kandinsky6SRDecodeStage, + Kandinsky6SRDenoisingStage, + Kandinsky6SREncodeStage, + Kandinsky6SRInputStage, + Kandinsky6SRLatentPrepStage, + Kandinsky6SROutputStage, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( TilePlan, ) @@ -72,19 +64,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k RESOLUTIONS, ) - -@pytest.fixture(scope="module", autouse=True) -def single_process_model_parallel(): - if not model_parallel_is_initialized(): - for key, value in dict( - MASTER_ADDR="127.0.0.1", - MASTER_PORT="29509", - RANK="0", - LOCAL_RANK="0", - WORLD_SIZE="1", - ).items(): - os.environ.setdefault(key, value) - maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) +pytestmark = pytest.mark.usefixtures("single_process_model_parallel") @pytest.fixture(autouse=True) @@ -93,22 +73,25 @@ def tiny_resolutions(monkeypatch): @pytest.mark.parametrize( - "bank_scales", - [("2x", "4x"), (), ("4x",)], + "bank_scales,nfe,num_steps", + [(("2x", "4x"), 2, 5), ((), None, 3), (("4x",), 3, 9)], ids=["lu_path", "pixel_path", "pixel_fallback"], ) -def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): - """encode -> latent-prep -> denoise -> decode -> output must give what - ``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank - that has no entry for the requested scale (which must fall back to the pixel path, not - fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and - 2.""" - vae, dit, bank, server_args = build_components(bank_scales) +def test_stage_chain_reproduces_the_pure_orchestration(bank_scales, nfe, num_steps): + """Cover LU/pixel fallback and scheduler reuse across chunks and requests.""" + scheduler = ( + PiflowScheduler(**dict(TINY_PIFLOW, nfe=nfe)) + if nfe is not None + else FlowMatchEulerDiscreteScheduler(shift=5.0) + ) + vae, dit, bank, server_args = build_components( + bank_scales, dict(TINY_DIT, out_visual_dim=12 if nfe is not None else 4) + ) video = random_video(9, 64, 128) - batch = make_request(video, tiles_batch_size=5) + batch = make_request(video, tiles_batch_size=5, num_steps=num_steps) pipeline = object.__new__(Kandinsky6SRPipeline) - pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None) + pipeline.modules = dict(vae=vae, transformer=dit, scheduler=scheduler) if bank is not None: pipeline.modules["latent_upscaler"] = bank pipeline._stages, pipeline._stage_name_mapping = [], {} @@ -148,7 +131,14 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan) - batch = denoise.forward(batch, server_args) + dit_calls = [] + hook = dit.register_forward_hook(lambda *args: dit_calls.append(1)) + with patch.object(denoise, "progress_bar", wraps=denoise.progress_bar) as progress: + batch = denoise.forward(batch, server_args) + hook.remove() + assert len(dit_calls) == 3 * (nfe or num_steps) + assert progress.call_args.kwargs["total"] == len(dit_calls) + assert denoise.scheduler is scheduler batch = decode.forward(batch, server_args) assert len(batch.extra[SR_TILES_KEY]) == 12 result = output.forward(batch, server_args) @@ -159,9 +149,10 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): arch=arch, tiling_scale=2, seed=42, - num_steps=5, + num_steps=num_steps, tiles_batch_size=5, tile_min_overlap=0.2, + scheduler=scheduler, ) expected = super_resolve( video, @@ -170,7 +161,7 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): dit=dit, dit_spec=build_dit_spec(dit), spec=spec, - scheduler=effective_scheduler(spec, None), + scheduler=scheduler, device=next(dit.parameters()).device, upscale_fn=partial(bank.upscale, scale=2) if use_lu else None, lu_dtype=module_dtype(bank) if use_lu else None, @@ -184,6 +175,24 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra +def test_mismatched_sampler_fails_before_encoding_tiles(): + vae, dit, _, server_args = build_components(()) + stage = make_stage( + Kandinsky6SRLatentPrepStage, + server_args, + vae, + dit, + None, + FlowMatchEulerDiscreteScheduler(shift=5.0), + ) + batch = make_request(random_video(9, 64, 96)) + with patch.object(vae, "encode", wraps=vae.encode) as encode: + with pytest.raises(ValueError, match="head.*same checkpoint"): + stage.forward(batch, server_args) + encode.assert_not_called() + assert SR_VIDEO_KEY in batch.extra + + @pytest.mark.parametrize( "role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER] ) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py index 6fe9f05341..e622a4661e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py @@ -9,7 +9,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k SamplingSpec, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - build_chunk_latent, + plan_tiles, + prepare_tile_latents, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( compute_tile_grid_even, @@ -63,22 +64,30 @@ def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype): tiles_batch_size=1, seed=1, num_steps=5, + is_piflow=False, tile_min_overlap=0.2, visual_size=512, scale_factor=(1.0, 1.0, 1.0), - scheduler_scale=5.0, lq_noise_scale=0.7, lq_noise_type="linear", lq_channel_noise_scale=0.0, cap_noise_timestep=False, - piflow=None, ) tile = torch.randn(4, 4, 4, 4, dtype=torch.float64) - result = build_chunk_latent( - [tile], - seed=1, + plan = plan_tiles( + frame_hw=(64, 64), + visual_size=512, + tiling_scale=2, + tile_min_overlap=0.2, + resolutions={512: [(128, 128)]}, + ) + result = prepare_tile_latents( + tile, + plan, + tile_encoder=lambda tile: tile.permute(0, 2, 3, 1), + latent_path=True, dit_spec=dit_spec, spec=spec, device=torch.device("cpu"), - ) + )[0] assert result.dtype == (dit_dtype or tile.dtype) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py index 696518f141..8716dfebc5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py @@ -15,28 +15,39 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( ) from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( - Kandinsky6SRInputStage, -) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( SR_TILING_SCALE_KEY, SR_VIDEO_KEY, ) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import ( + Kandinsky6SRInputStage, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( FrameSelector, decode_clip, extract_source_audio, to_output_video, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( - clip_to_aligned_frames, - resample_to_target_fps, -) FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000] SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120] +def clip_to_aligned_frames(video): + video = video[:121] + if not len(video): + raise ValueError("empty video") + return video[: 1 + 8 * ((len(video) - 1) // 8)] + + +def _reference_resample(video, fps): + if abs(fps - 24) < 1.5 or fps < 24: + return video, round(fps) + step = fps / 24 + indices = [round(i * step) for i in range(int(len(video) / step))] + return video[[i for i in indices if i < len(video)]], 24 + + def _stream_select(total: int, fps: float) -> tuple[list[int], int]: """Run the streaming selector over frames ``0 .. total - 1`` (early stop included).""" selector = FrameSelector(fps) @@ -54,7 +65,7 @@ def _stream_select(total: int, fps: float) -> tuple[list[int], int]: def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None: ids = torch.arange(total).view(total, 1, 1, 1) - resampled, effective_fps = resample_to_target_fps(ids, fps) + resampled, effective_fps = _reference_resample(ids, fps) try: clipped = clip_to_aligned_frames(resampled) except ValueError: @@ -147,7 +158,7 @@ def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames): path = tmp_path / "clip.mp4" write_test_video(path, frames=frames, fps=fps) full, source_fps = _decode_everything(path) - expected, expected_fps = resample_to_target_fps(full, source_fps) + expected, expected_fps = _reference_resample(full, source_fps) expected = clip_to_aligned_frames(expected) clip = decode_clip(str(path)) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py index 95d014a2c7..d9f330cf97 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py @@ -4,6 +4,7 @@ import math from types import SimpleNamespace +import numpy as np import PIL.Image import pytest import torch @@ -81,6 +82,35 @@ class ImageVAE: return SimpleNamespace(sample=lambda generator=None: self.latent) +@pytest.mark.parametrize("source_size", [(1920, 1080), (1080, 1920), (288, 192)]) +@pytest.mark.parametrize("tensor_input", [False, True], ids=["pil", "tensor"]) +def test_image_preprocessing_preserves_aspect_and_center_crops( + source_size, tensor_input +): + src_w, src_h = source_size + height, width = 64, 96 + pixels = np.full((src_h, src_w, 3), 128, dtype=np.uint8) + pixels[..., 0] = np.linspace(0, 255, src_w, dtype=np.uint8)[None, :] + pixels[..., 1] = np.linspace(0, 255, src_h, dtype=np.uint8)[:, None] + image = PIL.Image.fromarray(pixels) + + # independent PIL oracle for the reference resize + center-crop geometry + scale = min(src_h / height, src_w / width) + new_h, new_w = int(src_h / scale), int(src_w / scale) + top, left = (new_h - height) // 2, (new_w - width) // 2 + expected = image.resize((new_w, new_h), PIL.Image.Resampling.BILINEAR).crop( + (left, top, left + width, top + height) + ) + expected = torch.from_numpy(np.array(expected)).permute(2, 0, 1).float() / 127.5 - 1 + source = ( + torch.from_numpy(pixels).permute(2, 0, 1).float() / 127.5 - 1 + if tensor_input + else image + ) + actual = Kandinsky6ImageEncodingStage._preprocess(source, height, width) + torch.testing.assert_close(actual, expected.unsqueeze(0), atol=2 / 255, rtol=0) + + @pytest.mark.parametrize("batch_size", [1, 2]) def test_image_request_broadcasts_one_reference_to_each_output(server_args, batch_size): request = VideoGenerationsRequest(