From 16a23a672b0688c415fa81b056e24bf7b7db320d Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 6 Oct 2026 22:13:29 +0800 Subject: [PATCH] [diffusion] model: support Kandinsky 6 (#42743) Co-authored-by: Mick Qian Co-authored-by: Lev Novitskiy <57654885+leffff@users.noreply.github.com> Co-authored-by: Kirill Co-authored-by: Artem Borisov --- .github/workflows/lint.yml | 4 +- .../diffusion/Kandinsky/Kandinsky6-SR.mdx | 282 ++++ .../diffusion/Kandinsky/Kandinsky6.mdx | 345 ++++ docs/custom.css | 9 +- docs/docs.json | 8 + docs/scripts/tests/kandinsky6-config.test.mjs | 305 ++++ .../configs/Kandinsky/kandinsky6-sr.jsx | 245 +++ .../snippets/configs/Kandinsky/kandinsky6.jsx | 325 ++++ docs/src/snippets/diffusion/model-catalog.jsx | 18 + python/sglang/kernels/ops/diffusion/README.md | 4 + .../sglang/kernels/ops/diffusion/__init__.py | 16 + .../diffusion/modulate/residual_gate_fp32.py | 96 ++ .../kernels/ops/diffusion/rope/matrix_rope.py | 111 ++ .../configs/models/dits/kandinsky6.py | 147 ++ .../configs/models/dits/kandinsky6_sr.py | 269 ++++ .../models/encoders/kandinsky6_reason1.py | 88 + .../configs/models/upsamplers/__init__.py | 7 + .../models/upsamplers/kandinsky6_sr.py | 225 +++ .../configs/models/vaes/kandinsky6_audio.py | 37 + .../configs/models/vaes/kandinsky6_sr.py | 96 ++ .../configs/pipeline_configs/kandinsky6.py | 248 +++ .../configs/pipeline_configs/kandinsky6_sr.py | 76 + .../configs/sample/kandinsky6.py | 65 + .../configs/sample/kandinsky6_sr.py | 124 ++ .../sample/kandinsky6_sr_resolution.py | 96 ++ .../configs/sample/sampling_params.py | 40 + .../runtime/cache/cache_dit_integration.py | 33 +- .../device_communicators/ipc_a2a.py | 12 + .../entrypoints/diffusion_generator.py | 6 +- .../runtime/entrypoints/openai/video_api.py | 13 +- .../runtime/layers/attention/layer.py | 14 +- .../runtime/layers/lora/linear.py | 68 +- .../multimodal_gen/runtime/layers/usp.py | 24 +- .../component_loaders/component_loader.py | 11 +- .../latent_upscaler_loader.py | 80 + .../runtime/managers/gpu_worker.py | 2 + .../layerwise_offload_components.py | 1 + .../runtime/managers/scheduler.py | 1 + .../runtime/models/dits/kandinsky6.py | 1431 +++++++++++++++++ .../runtime/models/dits/kandinsky6_sr.py | 430 +++++ .../runtime/models/encoders/clip.py | 3 + .../models/schedulers/kandinsky6_piflow.py | 314 ++++ .../kandinsky6_sr_latent_upscaler.py | 336 ++++ .../{minimax_h3_audio_vae => }/alias_free.py | 12 + .../runtime/models/vaes/common.py | 14 +- .../runtime/models/vaes/hunyuanvae.py | 61 +- .../runtime/models/vaes/kandinsky6_audio.py | 120 ++ .../kandinsky6_audio_vae/THIRD_PARTY_LICENSES | 31 + .../vaes/kandinsky6_audio_vae/__init__.py | 8 + .../vaes/kandinsky6_audio_vae/bigvgan.py | 350 ++++ .../kandinsky6_audio_vae/mel_converter.py | 69 + .../vaes/kandinsky6_audio_vae/mmaudio_vae.py | 854 ++++++++++ .../runtime/models/vaes/kandinsky6_sr_vae.py | 656 ++++++++ .../vaes/minimax_h3_audio_vae/bigvgan.py | 8 +- .../runtime/pipelines/kandinsky6_pipeline.py | 122 ++ .../pipelines/kandinsky6_sr_pipeline.py | 103 ++ .../runtime/pipelines_core/schedule_batch.py | 6 + .../pipelines_core/stages/denoising.py | 16 +- .../kandinsky6/__init__.py | 16 + .../kandinsky6/decoding.py | 94 ++ .../kandinsky6/denoising.py | 442 +++++ .../kandinsky6/image_encoding.py | 214 +++ .../kandinsky6/latent_preparation.py | 151 ++ .../kandinsky6_sr/__init__.py | 8 + .../kandinsky6_sr/decode_stage.py | 78 + .../kandinsky6_sr/denoising_stage.py | 137 ++ .../kandinsky6_sr/encode_stage.py | 65 + .../kandinsky6_sr/input_stage.py | 95 ++ .../kandinsky6_sr/latent_prep_stage.py | 162 ++ .../kandinsky6_sr/latents.py | 134 ++ .../kandinsky6_sr/output_stage.py | 72 + .../kandinsky6_sr/run_spec.py | 221 +++ .../kandinsky6_sr/sampling.py | 191 +++ .../kandinsky6_sr/tiled.py | 413 +++++ .../kandinsky6_sr/tiling.py | 248 +++ .../kandinsky6_sr/video_io.py | 190 +++ .../kandinsky6_sr/video_utils.py | 136 ++ .../pipelines_core/stages/text_encoding.py | 38 +- .../runtime/utils/hf_diffusers_utils.py | 35 +- .../test/server/test_server_kandinsky6.py | 250 +++ .../test/server/test_server_kandinsky6_sr.py | 172 ++ .../unit/kandinsky6_sr_release_configs.py | 46 + .../unit/kandinsky6_sr_tiny_components.py | 259 +++ .../test/unit/test_cache_dit_integration.py | 93 +- .../test/unit/test_cache_dit_per_request.py | 45 +- .../unit/test_hunyuan_vae_frame_attention.py | 151 ++ .../test/unit/test_ipc_a2a_lifecycle.py | 37 + .../test/unit/test_kandinsky6_arch_config.py | 114 ++ .../test_kandinsky6_audio_vae_weight_norm.py | 96 ++ .../test/unit/test_kandinsky6_bf16_dtype.py | 502 ++++++ .../test/unit/test_kandinsky6_clip_loading.py | 157 ++ ...st_kandinsky6_image_conditioning_resize.py | 127 ++ .../unit/test_kandinsky6_layerwise_offload.py | 271 ++++ .../test/unit/test_kandinsky6_lora.py | 88 + .../unit/test_kandinsky6_pipeline_contract.py | 346 ++++ .../test/unit/test_kandinsky6_registry.py | 207 +++ .../test/unit/test_kandinsky6_rope.py | 123 ++ .../unit/test_kandinsky6_sequence_parallel.py | 166 ++ .../test/unit/test_kandinsky6_sr_config.py | 305 ++++ .../test_kandinsky6_sr_decode_parallel.py | 210 +++ .../unit/test_kandinsky6_sr_http_request.py | 290 ++++ .../test/unit/test_kandinsky6_sr_loading.py | 467 ++++++ ..._kandinsky6_sr_official_release_configs.py | 369 +++++ .../test_kandinsky6_sr_reference_parity.py | 829 ++++++++++ .../test_kandinsky6_sr_sampler_selection.py | 253 +++ .../test_kandinsky6_sr_sequence_parallel.py | 152 ++ .../test/unit/test_kandinsky6_sr_stages.py | 353 ++++ .../test/unit/test_kandinsky6_sr_tiling.py | 84 + .../test/unit/test_kandinsky6_sr_video_io.py | 240 +++ .../test/unit/test_kandinsky6_stages.py | 161 ++ .../test/unit/test_lora_pipeline.py | 42 +- .../test/unit/test_usp_ring_tail_pad.py | 51 +- .../test/unit/test_vae_tiled_decode_groups.py | 66 + .../kernels/ops/diffusion/test_matrix_rope.py | 91 ++ .../ops/diffusion/test_residual_gate_fp32.py | 108 ++ 115 files changed, 18816 insertions(+), 140 deletions(-) create mode 100644 docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx create mode 100644 docs/cookbook/diffusion/Kandinsky/Kandinsky6.mdx create mode 100644 docs/scripts/tests/kandinsky6-config.test.mjs create mode 100644 docs/src/snippets/configs/Kandinsky/kandinsky6-sr.jsx create mode 100644 docs/src/snippets/configs/Kandinsky/kandinsky6.jsx create mode 100644 python/sglang/kernels/ops/diffusion/modulate/residual_gate_fp32.py create mode 100644 python/sglang/kernels/ops/diffusion/rope/matrix_rope.py create mode 100644 python/sglang/multimodal_gen/configs/models/dits/kandinsky6.py create mode 100644 python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/configs/models/encoders/kandinsky6_reason1.py create mode 100644 python/sglang/multimodal_gen/configs/models/upsamplers/__init__.py create mode 100644 python/sglang/multimodal_gen/configs/models/upsamplers/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_audio.py create mode 100644 python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py create mode 100644 python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/configs/sample/kandinsky6.py create mode 100644 python/sglang/multimodal_gen/configs/sample/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/configs/sample/kandinsky6_sr_resolution.py create mode 100644 python/sglang/multimodal_gen/runtime/loader/component_loaders/latent_upscaler_loader.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py create mode 100644 python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py rename python/sglang/multimodal_gen/runtime/models/vaes/{minimax_h3_audio_vae => }/alias_free.py (88%) create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/THIRD_PARTY_LICENSES create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/__init__.py create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mel_converter.py create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py create mode 100644 python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_pipeline.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/__init__.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/image_encoding.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/latent_preparation.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py create mode 100644 python/sglang/multimodal_gen/test/server/test_server_kandinsky6.py create mode 100644 python/sglang/multimodal_gen/test/server/test_server_kandinsky6_sr.py create mode 100644 python/sglang/multimodal_gen/test/unit/kandinsky6_sr_release_configs.py create mode 100644 python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_hunyuan_vae_frame_attention.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_arch_config.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_clip_loading.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_lora.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_registry.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_vae_tiled_decode_groups.py create mode 100644 test/registered/kernels/ops/diffusion/test_matrix_rope.py create mode 100644 test/registered/kernels/ops/diffusion/test_residual_gate_fp32.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2041ea817a..7c30400012 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -74,7 +74,9 @@ jobs: node-version: "20" - name: Check cookbook authoring contracts - run: node docs/scripts/check_cookbook_configs.mjs + run: | + node docs/scripts/check_cookbook_configs.mjs + node --test docs/scripts/tests/*.test.mjs - name: Test PR health action run: node --test .github/actions/check-pr-test-health/action.test.cjs diff --git a/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx b/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx new file mode 100644 index 0000000000..fc6fdd9f48 --- /dev/null +++ b/docs/cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx @@ -0,0 +1,282 @@ +--- +title: Kandinsky6 SR +description: "Deploy Kandinsky 6 video super-resolution with x2, x4 or x2.25 upscaling, source audio and explicit decode speed, memory and numerical tradeoffs." +--- + +import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx'; +import { Deployment } from '/src/snippets/_deployment.jsx'; +import { config } from '/src/snippets/configs/Kandinsky/kandinsky6-sr.jsx'; + + + +## 1. Quick start + +Use Linux with NVIDIA CUDA and a checkout containing this implementation; it is +not yet in a released package or unmodified release image. Authenticate an account +with checkpoint access using `hf auth login`, or supply `HF_TOKEN` to the server. +From the repo root: + +```bash Install +uv pip install -e "python[diffusion]" --prerelease=allow +``` + +The picker uses the distilled two-step checkpoint. Four-GB300 whole-tile decode +is the output-preserving choice measured on multi-tile clips; one GPU suffices +for a functional check when it has enough memory. The picker also includes +explicitly **extrapolated** Hopper and RTX 5090 starting points, not measured +speed or memory guarantees. HTTP verification is scoped separately from offline +measurements. See the [installation guide](/docs/sglang-diffusion/installation) +for platform setup. + + + +The request uploads a local video and returns a job `id`. Poll +`GET /v1/videos/{id}`, then download `GET /v1/videos/{id}/content` when completed. +Use the shared [polling and download example](/cookbook/diffusion/Kandinsky/Kandinsky6#5-download-a-completed-video). +For JSON requests, `video_path` refers to a file on the **server**, not the client. + +## 2. Model capabilities + +Kandinsky 6 VSR upscales a source video without a prompt. It encodes the source +with a causal KVAE, denoises overlapping tiles and blends their decoded outputs. +Unlike the [generation model](/cookbook/diffusion/Kandinsky/Kandinsky6), it retains +the source soundtrack rather than generating audio. Audio is converted to mono +44.1 kHz and trimmed to the processed video duration. + +Use `kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers` for x2, x4 or +x2.25 upscaling of up to 121 frames. Its checkpoint-selected PiFlow scheduler +uses two DiT calls per tile; changing `num_inference_steps` does not increase +quality or the number of evaluations. The native path uses dense attention, +not the checkpoint's requested NABLA sparse attention, so this is not a claim +of numerical parity with the sparse reference. + + +The tested non-distilled `Kandinsky-6.0-VSR-5s-Diffusers` revision +`d6ddfabc62b3920b6da6487e4f444d37c591d5f4` declares a flow-Euler scheduler but +contains a 640-wide output head instead of the required 64. It is not supported +as-is. Do not change the scheduler to bypass this error; use the distilled model. + + +## 3. Offline usage + +No prompt is required. A single-GPU example: + +```bash Generate +sglang generate \ + --model-path kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers \ + --video-path /path/to/input.mp4 \ + --sr-resolution-scale 2 \ + --num-inference-steps 2 \ + --seed 42 --save-output --output-path ./sr_outputs +``` + + +```python Generate +from sglang.multimodal_gen import DiffGenerator + +generator = DiffGenerator.from_pretrained( + model_path="kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", + num_gpus=1, +) +try: + result = generator.generate(sampling_params_kwargs={ + "video_path": "/path/to/input.mp4", + "sr_resolution_scale": 2, + "num_inference_steps": 2, + "seed": 42, + "output_path": "sr_outputs", + "save_output": True, + }) + print(result.output_file_path) +finally: + generator.shutdown() +``` + + +### Request parameters + +- `sr_resolution_scale`: `2`, `4` or `2.25` (default). The x2.25 path first + enlarges pixels x1.125, then processes x2 tiles. +- `sr_tiles_batch_size`: default `1`. More tiles per DiT call need more activation + memory. Each chunk is seeded by its first tile index, so changing this value + changes the noise and output. It is **not a lossless batching optimization**. +- `sr_tile_min_overlap`: default `0.2`. Changing overlap changes tile geometry + and blending, not just performance. +- `sr_target_resolution`: optional `hd`, `fullhd`, `2k` or `WxH`; only downscales + the generated result. `sr_target_resize_mode` is `fit` (default) or `exact`. +- `seed`: default `42`. Compare identical seeds, tile batching, source frames and + attention policy when evaluating a decode optimization. + +The input must fit at least one tile. A tested landscape source is 384 x 256. +The default filename is `_srx_.mp4`; override it +with `--output-file-name`. + +## 4. Decode speed, memory and numerics + +These are **VAE decode choices**, independent of DiT TP/Ulysses/Ring: + + + + + + + + +
DecodeHot E2EPeak reservedOutput versus serial
Serial45.26 s47.55 GiBReference
Whole-tile parallel21.73 s47.55 GiBRGB and audio exact in this test
Spatial shard18.01 s24.37 GiBRGB differs; audio exact
+ +Setup: four GB300 GPUs (277.5 GiB each, NV18 links), host RAM 955.7 GiB with an +860 GiB cgroup limit; BF16/FA, Ulysses4, resident weights, no compile or startup +warmup. PyTorch 2.14.1+cu130, Transformers 5.17.0, Diffusers 0.37.0. Distilled +checkpoint revision `3e3e10bcd23659eeeb23cac740258ee4fc14aa7d`; seed 42, tile +batch 1, two steps per tile. A generated kitchen clip, 768 x 512 x 121 frames at +24 fps, was upscaled x2 to 1536 x 1024. Times are medians of three hot requests +after one initial request, including MP4 saving and excluding model loading. +Memory is PyTorch request peak reserved on the output rank, not loading, NVML or +the maximum across ranks. Host memory is not claimed unchanged. + +**Whole-tile parallel** assigns complete existing tiles to ranks and preserves +their temporal caches and blend order. It matched serial RGB and raw audio +exactly on the tested workloads **within the same topology**. Select +`--vae-config.use-parallel-tiling true`. A single tile falls back to serial. +Communication can outweigh savings for small clips or slow interconnects. + +**Spatial shard is not bitwise lossless.** It splits each tile's height, using +halo exchanges and the original temporal cache. Changed convolution shapes +change rounding. In the experiment above, all-frame RGB PSNR was **57.51 dB**, +minimum SSIM over the first/middle/last frames **0.99921**, mean absolute +difference **0.1132/255**, and maximum difference **4/255**. Audio was exact. +These measurements are not a guarantee for other videos or perceptual-quality +certification. Select it only after validating the output for your workload: + +```bash Spatial decode flags +--vae-config.use-parallel-decode true --vae-config.parallel-decode-mode spatial_shard +``` + +Both modes are opt-in runtime features. The picker selects whole-tile parallel +for its multi-tile recipe; it does not change runtime defaults. If both flags +are enabled, spatial sharding takes precedence. One GPU or a tile too short to +split retains replicated decoding. Neither mode changes the tile geometry. + +### RTX PRO 6000 over PCIe + +| GPUs / placement | Decode | Hot E2E | Sampled GPU peak | +|---|---|---|---| +| 1 / resident | Serial | 134.09 s | 48.91 GiB | +| 2 / resident, Ulysses2 | Serial | 122.11 s | 48.81 GiB | +| 2 / resident, Ulysses2 | Whole-tile | 83.04 s | 48.81 GiB | +| 2 / resident, Ulysses2 | Spatial | 73.62 s | 37.45 GiB | +| 1 / all-layerwise offload | Serial | 133.60 s | 37.43 GiB | + +Setup: two RTX PRO 6000 Blackwell Server Edition GPUs (96GB nominal), PCIe/SYS +across CPU sockets, **no NVLink**; 499.2 GiB host RAM, no cgroup cap. Driver +595.84, the same PyTorch/Transformers/Diffusers versions and checkpoint as above, +but **SDPA**, not FA. A separate generated 768 x 512 x 121-frame clip at 24 fps +was upscaled x2 with seed 42, tile batch 1 and two steps per tile. Each row uses +one initial request and three hot requests, no compile/startup warmup. E2E +includes MP4 saving and raw-frame return, excludes model loading. GPU memory is +the maximum **per-GPU NVML sample across ranks**, sampled every 0.5 seconds during +requests, not the sum or PyTorch reserved memory; brief peaks may be missed. +These are within-machine comparisons, not GB300-versus-RTX speed ratios. + +Ulysses2 alone reduced latency by only 8.9%. Whole-tile decode reduced it by +38.1% versus one GPU, with **exact RGB/audio** in this test. Spatial reached +45.1% lower latency, but RGB differed: PSNR **57.54 dB**, minimum sampled SSIM +**0.99919**, maximum difference **4/255**; audio remained exact. All five +configurations reproduced their own outputs across repeated requests. +Layerwise offload reduced memory with essentially unchanged latency here; this +does not make offload free on other workloads or slower host links. + +A separate **memory-budget probe**, still on RTX PRO 6000, completed the full +clip twice with all-layerwise offload and a **30 GiB PyTorch allocator cap**. +Sampled whole-GPU usage peaked at **31.03 GiB**, and RGB/audio matched the +resident baseline exactly. The cap excludes CUDA allocations outside PyTorch; +it is a test setting, not a deployment recommendation. This supports trying the +32GB recipe, but is **not a 5090 speed, memory-fit or quality measurement**. + +The complete HTTP regression also passed on one-GPU resident serial, Ulysses2 +resident whole-tile and one-GPU layerwise serial with the allocator cap: +six requests each covering JSON, uploads, x2/x4/x2.25, tile batch 2, +audio/silent inputs and repeated-request equality. Arbitrary videos still need +their own validation. + +### Small workloads do not necessarily scale + +On GB300, a 384 x 256 x 33-frame source upscaled x2 (one tile, serial decode, +otherwise the same BF16/FA/seed/step policy) took **1.77 s on one GPU, +1.72 s on Ulysses2 and 1.74 s on Ulysses4**. These are three-hot-request medians +including saving. The roughly 2% difference is not evidence of a useful speedup; +prefer one GPU for this workload. More tiles, not just more GPUs, create work +that whole-tile decoding can distribute. + +## 5. Memory and parallelism + +### Hardware and GPU counts + +| Hardware | Starting point | Evidence boundary | +|---|---|---| +| GB300 | Resident, FA; Ulysses4 + whole-tile for multi-tile clips | Measured offline; smaller 1/2/4-GPU workloads also tested | +| RTX PRO 6000 96GB | Resident, SDPA; compare 1 GPU with Ulysses2 | PCIe measurements, not a substitute for 5090 memory validation | +| H100 80GB / H200 141GB | Resident, FA; start with 1 GPU, then Ulysses2/4 | Extrapolated from native CUDA support; no Hopper SR measurement | +| B200 | Resident, FA; start with 1 GPU, then Ulysses2/4 | Extrapolated SR recipe; generation-model tests do not validate SR | +| RTX 5090 32GB | SDPA, layerwise offload, tile batch 1; short 384 x 256 input first | Extrapolated from the RTX memory-budget probe; no 5090 execution claim | + +Only compare GPU counts on the **same input, scale, tile batching and backend**. +For small single-tile clips, one GPU avoids communication overhead. For larger +multi-tile clips, try Ulysses2/4 with whole-tile decode; extra GPUs do not divide +replicated KVAE/latent-upscaler weights. Spatial decode can reduce activation +memory, but changes rounding. Offload needs sufficient host RAM as well as VRAM. +Do not extrapolate NVLink scaling to a PCIe system, especially across CPU sockets. + +Eight-GPU FA configurations are selectable as **Ulysses4 x Ring2**, not Ulysses8: +28 heads are not divisible by 8. This is an unmeasured topology, not a speedup +recommendation. The current RTX SDPA path has no Ring support, so the picker +rejects that combination. Start with fewer GPUs before trying an eight-GPU run. + +### Placement and topology + +DiT Ulysses, TP, Ring and TP x SP are supported. For example, use +`--num-gpus 4 --tp-size 1 --ulysses-degree 4` for Ulysses4, or +`--num-gpus 4 --tp-size 2 --ulysses-degree 2` for TP2 x Ulysses2. +TP times Ulysses must divide the checkpoint's 28 attention heads. +**TP and Ring can change output numerics independently of decode mode**; +whole-tile decode being exact does not make a TP/Ring topology change lossless. + +The decode group includes TP/SP ranks within each DP replica, not different +requests. KVAE encoding, latent upscaling and tile scheduling remain replicated. +For short two-step clips, extra GPUs can add communication without a speedup. + +Keep components resident when memory permits. For less GPU weight memory, use +`--layerwise-offload-components all`, or select `vae latent_upscaler` instead. +Temporal caches and non-block weights remain on GPU. Offload is compatible with +both decode modes, but consumes host RAM and adds transfers for tiles and temporal +segments. The assembled output also occupies host memory until saved. + +The **Server** tab also offers whole-component offload with +`--cpu-offload-components`, selectable offload components, and DiT prefetch and +resident-layer counts when streaming the DiT. Increasing those counts consumes +additional VRAM. Partial offload recipes, non-default layer counts and +FSDP (`--use-fsdp-inference true`) are **unverified options**, not measured +improvements. FSDP requires multiple GPUs; SDPA cannot be combined with Ring. +The default backend is FA on datacenter Blackwell/Hopper and SDPA on RTX. +Hardware starting points do not automatically adapt to every uploaded video's +memory demand. Request scale and tile batching do not require restarting the server. + +## 6. Limits + +- **Input:** at most 121 frames. Sources near 24 fps retain their frames; faster + videos are subsampled, slower videos retain their native rate. This can change + the source timing; SR is not a bit-preserving video/audio transcode. +- **Attention:** this native path is dense. Do not compare it with a NABLA sparse + reference and attribute every difference to parallel decoding. The latent + upscaler uses SDPA motion attention when `natten` is unavailable. +- **Deployment:** monolithic only. Disaggregated roles, `torch.compile`, LoRA, + direct latent inputs and KVAE-bridge chaining are not supported. +- **Loading:** component config and weight mismatches fail explicitly. For a + renamed local checkpoint, pass + `--model-id Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers` to retain SR routing. + +See the [compatibility matrix](/docs/sglang-diffusion/compatibility_matrix) for +other model families. The opt-in server regression is +`python/sglang/multimodal_gen/test/server/test_server_kandinsky6_sr.py`; it covers +uploads, JSON, all three scales, source audio and repeated requests, not official +pipeline parity. diff --git a/docs/cookbook/diffusion/Kandinsky/Kandinsky6.mdx b/docs/cookbook/diffusion/Kandinsky/Kandinsky6.mdx new file mode 100644 index 0000000000..9c6d9a0385 --- /dev/null +++ b/docs/cookbook/diffusion/Kandinsky/Kandinsky6.mdx @@ -0,0 +1,345 @@ +--- +title: Kandinsky6 +description: "Deploy Kandinsky 6.0 for joint text- and image-conditioned video and audio generation." +--- + +import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx'; +import { Deployment } from '/src/snippets/_deployment.jsx'; +import { config } from '/src/snippets/configs/Kandinsky/kandinsky6.jsx'; + + + +## 1. Quick start + +Use Linux with NVIDIA CUDA. This model is not yet in a released package; +use a checkout containing this implementation, not an unmodified release image. +Your Hugging Face account must have access to the checkpoint; authenticate with +`hf auth login` or supply `HF_TOKEN` in the server environment. Install from the repo root: + +```bash +uv pip install -e "python[diffusion]" --prerelease=allow +``` + +Start with Pro-distill and resident weights on B200, GB300, or RTX PRO 6000 (96 GB). +The picker separates +verified lightweight HTTP requests from full-size offline generation and +unverified custom topologies. See the [installation guide](/docs/sglang-diffusion/installation) +for platform setup. + + + +The request returns a video job ID, not the video bytes. Poll +`GET /v1/videos/{id}` until `status` is `completed`, then download +`GET /v1/videos/{id}/content` for the MP4 containing both video and audio. +Treat `failed` as an error; delete the job with `DELETE /v1/videos/{id}` when finished. +See [Download a completed video](#5-download-a-completed-video) for a complete polling example. + +## 2. Model capabilities + +Kandinsky 6.0 generates video and audio jointly from a text prompt, optionally +conditioned on a reference image. One DiT predicts both modalities at each +denoising step. A Reason1 encoder derived from Qwen2.5-VL supplies token features, +and CLIP supplies pooled text features. Both request modes use the same server; +image conditioning does not require switching checkpoints. + +Choose Pro-distill for the 10-step path, or Pro for 50-step classifier-free guidance. +The recipes cover 768 x 512, 121-frame clips at 24 fps. They use native SGLang +components, without calling a third-party model pipeline. Validation compares +against the initial SGLang implementation; it does not establish parity with +the unpublished Kandinsky6 Diffusers implementation. + +| Checkpoint | Scheduler | Default steps | Default guidance | +| --- | --- | ---: | ---: | +| `kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers` | PiFlow | 10 | 1.0 | +| `kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers` | Flow-match Euler | 50 | 5.0 | + +Pro-distill only accepts guidance 1.0 and does not encode a negative prompt. +The scheduler is loaded from the checkpoint. For a local directory with an +unrelated name, specify `--model-id Kandinsky-6.0-Pro-distill-5s-Diffusers` to +select its sampling defaults, or explicitly pass 10 steps and guidance 1.0. + +## 3. Offline generation + +The default request produces a muxed video and audio file: + +```bash +sglang generate \ + --model-path kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers \ + --performance-mode speed --attention-backend fa --warmup-mode off \ + --prompt "A woman chops vegetables in a sunlit kitchen, with soft jazz playing." \ + --seed 42 --save-output +``` + +Add `--image-path /path/to/reference.png` for image conditioning. +On RTX PRO 6000, use `--attention-backend torch_sdpa` in the CLI or +`attention_backend="torch_sdpa"` in Python. +For two B200 GPUs, add `--num-gpus 2 --tp-size 1 --ulysses-degree 2 --ring-degree 1`. +To use Pro, change the model path; its defaults select 50 steps and guidance 5.0. + +### Python API + +```python +from sglang.multimodal_gen import DiffGenerator + +generator = DiffGenerator.from_pretrained( + model_path="kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers", + num_gpus=1, + performance_mode="speed", + attention_backend="fa", + warmup_mode="off", +) +try: + result = generator.generate(sampling_params_kwargs={ + "prompt": "A woman chops vegetables in a sunlit kitchen, with soft jazz playing.", + "seed": 42, + "save_output": True, + "output_path": "kandinsky6_samples", + "return_file_paths_only": False, + "return_frames": True, + }) + print(result.output_file_path) +finally: + generator.shutdown() +``` + +`result.frames` contains RGB frames and `result.audio` contains the decoded waveform. +Add `image_path` to the sampling parameters for a reference image. Omit the two +return-data flags when only the output file is needed. + +## 4. Parallelism and memory + +- **Ulysses**: full-size Pro-distill generation on two B200 or four GB300 GPUs + matches single-GPU output on the same hardware, including every decoded RGB + frame and the audio waveform. + It shards the video stream; audio and text streams remain replicated. Video K/V + is gathered for audio-to-video attention. +- **CFG parallel**: Pro can distribute its conditional and unconditional branches + with `--num-gpus 2 --ulysses-degree 1 --enable-cfg-parallel`. + This is a different topology from Ulysses. It does not accelerate Pro-distill, + which has no guidance branch. +- **TP and Ring**: implemented, but full-model outputs differ from the single-GPU + baseline. Do not treat them as verified bitwise-equivalent modes. + Ring2, Ring4, and TP2 x Ring2 complete full-size Pro-distill generation on + GB300 and repeat exactly within each topology. Ring2 serving also covers + image conditioning, Cache-DiT, and dynamic/merged LoRA restoration. Use + `--num-gpus 2 --ulysses-degree 1 --ring-degree 2` to select Ring2; validate + both video and audio quality before deployment. +- **TP x SP**: four-GB300 TP2 x Ulysses2 full generation is tested. It reduces + weight memory but was slower than Ulysses4 and produced different video and + audio. Prefer Ulysses when memory permits; validate quality before deploying TP. +- **FSDP + Ulysses**: add `--use-fsdp-inference true` to shard DiT weights + without sharding linear reductions. Four-GB300 Ulysses4 generation matches + resident Ulysses4 in every decoded RGB frame and the audio waveform. This is + a lower-memory alternative with additional weight-gather latency, not a + faster default. Request-level Cache-DiT switching is also tested with FSDP. +- **Encoder parallelism**: `--encoder-parallel fold` shards Reason1 and CLIP + across the DiT replica, including a Ulysses-only replica. Four-GPU generation + is tested, but folding changes reductions and the generated video and audio. + Keep `auto` for the baseline recipes and validate quality before forcing + folding. See [Encoder parallelism](/docs/sglang-diffusion/encoder_parallel). +- **Offload**: keep components resident when memory allows. Use + `--cpu-offload-components transformer text_encoder vae` for whole-component + offload, or `--layerwise-offload-components all` for layerwise offload. + Layerwise offload includes both DiT text towers and the audio VAE's encoder, + decoder, and vocoder blocks. Use component-specific residency settings to + keep selected components resident. + Both require host RAM and add transfers; lower device memory does not imply + lower total memory or lower latency. + +The picker's **Server** tab exposes attention, weight placement and offload +components. With DiT layerwise offload, you can also tune prefetch depth and +resident layers using `--dit-offload-prefetch-size` and +`--dit-layerwise-resident-layers`. Increasing either uses more VRAM; these +non-default settings are unverified tuning choices, not faster recommendations. +Component offload moves whole components; layerwise offload streams supported +blocks. They are alternatives to FSDP in this picker, not additive presets. + +### Deployment limits + +Use monolithic serving. Encoder/denoiser/decoder disaggregation is not supported. +Breakable CUDA graphs are also not enabled for Kandinsky6: the server disables +`--enable-breakable-cuda-graph`. Its variable-length text path needs additional +handling to preserve the unpadded computation; same-shape graph experiments do +not establish support for arbitrary requests. Keep eager execution for the +recipes on this page. + +### Choosing a topology + +Pro-distill, batch 1, 768 x 512, 121 frames, 10 steps, BF16/FA, no offload or +compilation, on GB300 with 277.5 GiB per GPU: + +| Topology | Hot E2E (s) | Output-rank peak reserved (GiB) | Matches single-GPU RGB/audio | +| --- | ---: | ---: | --- | +| Single GPU | 33.95 | 80.36 | Reference | +| Ulysses4 | 11.33 | 80.91 | Exact | +| Ulysses4 + FSDP | 12.13 | 42.54 | Exact | +| TP2 x Ulysses2 | 13.99 | 50.97 | No | +| TP4 | 17.72 | 35.91 | No | + +Times are medians of three hot requests after one initial request, with startup +warmup disabled and default conditioning caching enabled. They include MP4 +saving but exclude model loading. Memory is PyTorch runtime peak reserved on +the output rank, not loading peak or the maximum across all GPUs. These results +do not establish output quality for TP; TP also changes the text encoder's +numerical path. + +For this workload, FSDP reduces measured memory by 47% versus resident Ulysses4 +at a 7% latency cost. Add it when weight memory is the constraint; keep ordinary +Ulysses when latency is the priority. + +### RTX PRO 6000 and SageAttention3 + +Pro-distill also runs with resident weights on one 96 GB RTX PRO 6000 Blackwell +Server Edition. Use `--attention-backend torch_sdpa` on this SM120 device: +the current platform resolves `fa` to SDPA, so requesting `fa` does not establish +a FlashAttention benchmark. + +With [SageAttention3](https://github.com/thu-ml/SageAttention/tree/d1a57a546c3d395b1ffcbeecc66d81db76f3b4b5/sageattention3_blackwell) +installed, `--component-attention-backends transformer=sage_attn_3` selects it +only for the DiT, keeping the encoders on SDPA. The tested upstream revision +requires SM120/121 and rejects B200 SM100. With PyTorch 2.14, its build required +`NVCC_APPEND_FLAGS=-std=c++20`; no kernel source or hardware guard was modified. + +Same full-size workload and timing method as above, on one RTX PRO 6000 with +driver 595.84, PyTorch 2.14.1+cu130, Transformers 5.17.0, and Diffusers 0.37.0: + +| DiT backend | Hot E2E (s) | Runtime peak reserved (GiB) | +| --- | ---: | ---: | +| SDPA | 133.91 | 80.34 | +| SageAttention3 | 105.33 | 82.05 | + +Sage3 reduced latency by 21.3% but changed the outputs substantially: all-frame +PSNR 11.97 dB, mean SSIM 0.375, and raw-audio waveform SNR -2.21 dB versus SDPA. +These are paired differences, not perceptual quality scores. Each backend was +repeatable across four requests, but Sage3 is **not lossless** and is not the +default recipe. Validate both video and audio before choosing it. + +### VAE decode + +The video VAE defaults to tiled decoding, including multi-GPU execution. Its +decode group is separate from the DiT's TP and CFG collectives. Explicit +`--vae-config.parallel-decode-mode spatial_shard` uses whole-clip decoding with +frame-causal attention computed without a quadratic whole-video mask. + +Whole-clip decoding still needs substantial activation memory. A 768 x 512, +121-frame run on two B200 GPUs completed with the transformer, both text encoders, +and audio VAE offloaded, but reached 170.4 GiB output-rank peak reserved. This is +not a lower-memory replacement for tiled decode. Tiled and whole-clip decoding +use different normalization and attention contexts and are not interchangeable +numerical baselines; keep tiled decoding for the validated default output. + +The audio VAE contains both the mel decoder and vocoder. Current checkpoints +include a complete vocoder copy inside `audio_vae`, so this pipeline does not +also load the redundant standalone `vocoder` component. + +CUDA matrix RoPE uses a fused kernel that preserves the eager FP32 rounding +boundaries. CPU and autograd paths retain the eager implementation. No sampling +parameter or sparse-attention approximation is introduced. +The checkpoint's 128-wide Q/K normalization uses native PyTorch FP32 fusion +during CUDA inference without changing the stored parameter dtype or offload +policy. Other head widths retain the original normalization path. + +### Cache-DiT + +Cache-DiT is opt-in per request through `enable_cache_dit` and `cache_dit_params` +in the Python sampling parameters or the JSON `/v1/videos` body. Set +`enable_cache_dit: false` to restore uncached execution without restarting. +See [Cache-DiT configuration](/docs/sglang-diffusion/cache_dit) +for the available knobs. + +The **Request** tab offers an opt-in Cache-DiT switch for both text and image +conditioning. It uses the runtime cache defaults, not a quality-calibrated +preset. A short request may finish before cache warmup ends and gain no speedup. + +The adapter caches both evolving video and audio streams together. Its dynamic +decision uses the video residual, not an independent audio-quality metric. +Skipping blocks is approximate and can change both outputs substantially; +validate video and audio on your own prompts before enabling it. It is not part +of the lossless defaults, and should not be combined with breakable CUDA graphs. + +### LoRA adapters + +Use adapters trained for the same Kandinsky6 checkpoint architecture. The native +loader accepts PEFT A/B weights with Diffusers or SGLang DiT layer names, including +the `transformer.` prefix. Both video and audio branches belong to `transformer`; +an adapter can change both outputs. + +Add `--lora-path /path/to/adapter.safetensors` to load an adapter at startup. +To switch an existing server to dynamic LoRA without restarting: + +```bash +curl --fail-with-body http://localhost:30000/v1/set_lora \ + -H "Content-Type: application/json" \ + -d '{ + "lora_nickname": "kandinsky6", + "lora_path": "/path/to/adapter.safetensors", + "target": "transformer", + "strength": 1.0, + "merge_mode": "dynamic" + }' +``` + +The adapter path must be readable by the server. `dynamic` applies the low-rank +update during inference; `merge` updates the base weights before inference. +These modes may differ numerically. To disable either mode and restore the base +model: + +```bash +curl --fail-with-body http://localhost:30000/v1/unmerge_lora_weights \ + -H "Content-Type: application/json" -d '{"target": "transformer"}' +``` + +Adapter selection is server-wide, not isolated per request. Coordinate changes +with other clients. Merging offloaded weights retains CPU snapshots of the +modified layers for exact restoration; budget for this host memory. +Validation uses synthetic adapters to check loading, +strength changes, repeatability, and exact restoration of video and audio; +it does not establish the quality of a community adapter. + +See the [model inventory](/docs/sglang-diffusion/compatibility_matrix) and the +[super-resolution recipe](/cookbook/diffusion/Kandinsky/Kandinsky6-SR) for the +separate video SR pipeline. + +## 5. Download a completed video + +Use the `id` returned by the picker's Request command. The same endpoints serve +joint generation and SR. Set the base URL to match your server port: + +```python Download +import json +import shutil +import time +import urllib.request + +base_url = "http://localhost:30000" +video_id = "REPLACE_WITH_RETURNED_ID" +deadline = time.monotonic() + 1800 + +while time.monotonic() < deadline: + with urllib.request.urlopen(f"{base_url}/v1/videos/{video_id}", timeout=30) as response: + job = json.load(response) + if job["status"] == "failed": + raise RuntimeError(job) + if job["status"] == "completed": + with urllib.request.urlopen( + f"{base_url}/v1/videos/{video_id}/content", timeout=120 + ) as response, open("output.mp4", "wb") as output: + shutil.copyfileobj(response, output) + break + time.sleep(2) +else: + raise TimeoutError("Video generation did not finish within 30 minutes") +``` + +After downloading, remove the server-side job when no longer needed: + +```bash Cleanup +curl --fail-with-body -X DELETE http://localhost:30000/v1/videos/REPLACE_WITH_RETURNED_ID +``` + +## 6. ComfyUI + +There is no Kandinsky6 integrated DiT executor in the +[ComfyUI plugin](/docs/sglang-diffusion/comfyui). The generic server video node +has not been validated for this model's joint audio/video output. Use the HTTP +or offline recipes above until that integration is verified. diff --git a/docs/custom.css b/docs/custom.css index fd4c9b6c81..76fb5d4f9f 100644 --- a/docs/custom.css +++ b/docs/custom.css @@ -552,8 +552,9 @@ html.dark table tbody tr:nth-child(even) td, } .sgd-builder-messages { margin-top: 12px; padding: 8px 10px; border-radius: 6px; background: color-mix(in oklch, var(--sgdb-warn) 9%, transparent); color: color-mix(in oklch, var(--sgdb-warn) 75%, var(--sgdb-ink)); } .sgd-builder-messages[data-state="error"] { background: color-mix(in oklch, var(--sgdb-error) 9%, transparent); color: color-mix(in oklch, var(--sgdb-error) 78%, var(--sgdb-ink)); } -.sgd-builder-messages p { margin: 0; font-size: 11px; line-height: 1.45; } -.sgd-builder-messages p + p { margin-top: 5px; } +/* Mintlify can render snippet paragraphs as span[data-as="p"]. */ +.sgd-builder-messages :is(p, [data-as="p"]) { display: block; margin: 0; font-size: 11px; line-height: 1.45; } +.sgd-builder-messages :is(p, [data-as="p"]) + :is(p, [data-as="p"]) { margin-top: 5px; } .sgd-builder-setting-layout, .sgd-builder-setting-list, .sgd-builder-setting-item { min-width: 0; } .sgd-builder-setting-item { border-bottom: 1px solid var(--sgdb-line); } @@ -587,11 +588,11 @@ html.dark table tbody tr:nth-child(even) td, .sgd-builder-context--rail { display: none; } .sgd-builder-context-heading { display: flex; align-items: start; justify-content: space-between; gap: 12px; } .sgd-builder-context-heading > div > span { font-size: 13px; font-weight: 590; } -.sgd-builder-context-heading p { margin: 4px 0 0; color: var(--sgdb-muted); font-size: 11px; line-height: 1.45; } +.sgd-builder-context-heading :is(p, [data-as="p"]) { display: block; margin: 4px 0 0; color: var(--sgdb-muted); font-size: 11px; font-weight: 400; line-height: 1.45; } .sgd-builder-context-heading > small { flex: none; color: var(--sgdb-muted); font-size: 10px; } .sgd-builder-context-options { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px; margin-top: 12px; } .sgd-builder-context-note { margin-top: 12px; padding-top: 10px; border-top: 1px solid var(--sgdb-line); } -.sgd-builder-context-note p { margin: 0; color: var(--sgdb-muted); font-size: 11px; line-height: 1.5; } +.sgd-builder-context-note :is(p, [data-as="p"]) { display: block; margin: 0; color: var(--sgdb-muted); font-size: 11px; line-height: 1.5; } .sgd-builder-context-note a { display: inline-flex; margin-top: 8px; diff --git a/docs/docs.json b/docs/docs.json index 5f7164d489..5ea9031386 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1607,6 +1607,14 @@ "cookbook/diffusion/MiniMax/MiniMax-H3" ] }, + { + "group": "Kandinsky", + "tag": "NEW", + "pages": [ + "cookbook/diffusion/Kandinsky/Kandinsky6", + "cookbook/diffusion/Kandinsky/Kandinsky6-SR" + ] + }, { "group": "LingBot Video", "tag": "NEW", diff --git a/docs/scripts/tests/kandinsky6-config.test.mjs b/docs/scripts/tests/kandinsky6-config.test.mjs new file mode 100644 index 0000000000..0211e3f6bc --- /dev/null +++ b/docs/scripts/tests/kandinsky6-config.test.mjs @@ -0,0 +1,305 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const read = (path) => readFileSync(new URL(`../../${path}`, import.meta.url), "utf8"); +const load = async (name) => (await import("data:text/javascript," + encodeURIComponent( + read(`src/snippets/configs/Kandinsky/${name}.jsx`), +))).config; +const generation = await load("kandinsky6"); +const sr = await load("kandinsky6-sr"); +const selection = (config, overrides = {}) => ({ + ...config.commandBuilder.defaultSelection, + ...Object.fromEntries(config.overlayDims.map((dim) => [dim.id, dim.default])), + ...overrides, +}); +const flags = (config, s) => [ + ...config.commandBuilder.resolveDeployment(s).flags, + ...config.overlayDims.flatMap((dim) => { + if (dim.showWhen && !dim.showWhen(s)) return []; + const option = dim.options.find((option) => option.id === s[dim.id]); + if (!option || (typeof option.disabled === "function" ? option.disabled(s) : option.disabled)) return []; + return (typeof option.flags === "function" ? option.flags(s) : option.flags) || []; + }), +]; + +test("RTX recipe has a renderable hardware entry and uses SDPA", () => { + assert.deepEqual(generation.hardware.find((hw) => hw.id === "rtxpro6000"), { + id: "rtxpro6000", label: "RTX PRO 6000", vram: "96GB", vendor: "blackwell", + }); + assert.ok(flags(generation, selection(generation, { hw: "rtxpro6000" })) + .includes("--attention-backend torch_sdpa")); + const invalid = generation.commandBuilder.resolveDeployment(selection(generation, { + hw: "rtxpro6000", gpus_per_node: 2, topology_mode: "manual", + tp_size: 1, ulysses_degree: 1, ring_degree: 2, + })).builder; + assert.equal(invalid.verification.serve, "error"); + assert.match(invalid.errors.join(" "), /SDPA.*Ring/); +}); + +test("placement verification is scoped; single-GPU FSDP is invalid", () => { + const builder = generation.commandBuilder; + assert.ok(builder.resolveDeployment(selection(generation, { placement: "fsdp" })).builder.errors.length); + const s = selection(generation, { hw: "gb300", gpus_per_node: 4, placement: "fsdp" }); + assert.equal(builder.resolveDeployment(s).builder.verification.serve, "verified"); + assert.equal(builder.resolveDeployment({ ...s, placement: "offload" }).builder.verification.serve, "unverified"); + assert.ok(flags(generation, s).includes("--use-fsdp-inference true")); +}); + +test("checkpoint and conditioning choices change the correct command", () => { + const s = selection(generation, { weights: "pro", mode: "image", workload: "smoke" }); + assert.ok(flags(generation, s).some((flag) => flag.endsWith("Kandinsky-6.0-Pro-5s-Diffusers"))); + assert.match(generation.curl(s), /input_reference=@/); + assert.match(generation.curl(s), /num_inference_steps=2/); + assert.doesNotMatch(generation.curl({ ...s, mode: "text" }), /input_reference/); +}); + +for (const config of [generation, sr]) { + test(`${config.modelName}: invalid and custom topologies are not verified`, () => { + const s = selection(config, { gpus_per_node: 3 }); + const invalid = config.commandBuilder.resolveDeployment(s).builder; + assert.ok(invalid.errors.length); + assert.equal(invalid.verification.serve, "error"); + const custom = config.commandBuilder.resolveDeployment(selection(config, { + gpus_per_node: 4, topology_mode: "manual", tp_size: 4, ulysses_degree: 1, ring_degree: 1, + })).builder; + assert.equal(custom.errors.length, 0); + assert.equal(custom.verification.serve, "unverified"); + assert.match(custom.warnings.join(" "), /TP/); + }); + + test(`${config.modelName}: dense backend overrides and Ring incompatibility`, () => { + for (const attention of ["platform", "fa", "torch_sdpa"]) { + const s = selection(config, { attention }); + const command = flags(config, s); + assert.equal(command.filter((flag) => flag.startsWith("--attention-backend ")).length, 1); + assert.ok(command.includes(`--attention-backend ${attention === "platform" ? "fa" : attention}`)); + } + const invalid = config.commandBuilder.resolveDeployment(selection(config, { + attention: "torch_sdpa", topology_mode: "manual", gpus_per_node: 2, + tp_size: 1, ulysses_degree: 1, ring_degree: 2, + })).builder; + assert.equal(invalid.verification.serve, "error"); + assert.match(invalid.errors.join(" "), /SDPA.*Ring/); + }); + + test(`${config.modelName}: each component selection maps to one offload policy`, () => { + const dim = config.overlayDims.find((dim) => dim.id === "offload_components"); + for (const placement of ["component", "offload"]) { + for (const { id: offload_components } of dim.options) { + const s = selection(config, { placement, offload_components }); + const command = flags(config, s); + assert.ok(dim.showWhen(s)); + const prefix = placement === "component" ? "--cpu-offload-components" : "--layerwise-offload-components"; + assert.deepEqual(command.filter((flag) => /--(cpu|layerwise)-offload-components/.test(flag)), [ + `${prefix} ${offload_components}`, + ]); + assert.ok(!command.includes("--use-fsdp-inference true")); + assert.equal(config.commandBuilder.resolveDeployment(s).builder.verification.serve, "unverified"); + } + } + }); + + test(`${config.modelName}: DiT tuning is visible and emitted only when applicable`, () => { + const s = selection(config, { placement: "offload", offload_components: "transformer", prefetch: 2, resident_layers: 4 }); + const tuning = config.overlayDims.filter((dim) => ["prefetch", "resident_layers"].includes(dim.id)); + assert.equal(tuning.length, 2); + assert.ok(tuning.every((dim) => dim.showWhen(s) && !dim.verifiedWhen(s))); + assert.ok(flags(config, s).includes("--dit-offload-prefetch-size 2")); + assert.ok(flags(config, s).includes("--dit-layerwise-resident-layers 4")); + assert.equal(config.commandBuilder.resolveDeployment(s).builder.resolvedSettings.resident_layers, "4 layers"); + for (const overrides of [ + { placement: "resident" }, { placement: "fsdp" }, { placement: "component" }, + { offload_components: config === sr ? "vae latent_upscaler" : "text_encoder" }, + ]) { + const hidden = { ...s, ...overrides }; + assert.ok(tuning.every((dim) => !dim.showWhen(hidden))); + assert.ok(!flags(config, hidden).some((flag) => /--dit-(offload-prefetch-size|layerwise-resident-layers)/.test(flag))); + } + const defaults = selection(config, { placement: "offload" }); + assert.ok(tuning.every((dim) => dim.verifiedWhen(defaults))); + assert.ok(!flags(config, defaults).some((flag) => /--dit-(offload-prefetch-size|layerwise-resident-layers)/.test(flag))); + }); +} + +test("Sage3 is a DiT-only, approximate RTX override, not a verified recipe", () => { + const s = selection(generation, { hw: "rtxpro6000", attention: "sage_attn_3" }); + const command = flags(generation, s); + assert.ok(command.includes("--attention-backend torch_sdpa")); + assert.ok(command.includes("--component-attention-backends transformer=sage_attn_3")); + const builder = generation.commandBuilder.resolveDeployment(s).builder; + assert.equal(builder.errors.length, 0); + assert.equal(builder.verification.serve, "unverified"); + assert.match(builder.warnings.join(" "), /approximate.*PSNR 11.97.*SNR -2.21/); + for (const hw of ["b200", "gb300"]) { + const invalid = generation.commandBuilder.resolveDeployment({ ...s, hw }).builder; + assert.equal(invalid.verification.serve, "error"); + assert.match(invalid.errors.join(" "), /SM120/); + } + assert.match(generation.commandBuilder.resolveDeployment({ + ...s, topology_mode: "manual", gpus_per_node: 2, ulysses_degree: 1, ring_degree: 2, + }).builder.errors.join(" "), /Sage3.*Ring/); + assert.match(generation.commandBuilder.resolveDeployment({ ...s, attention: "fa" }) + .builder.errors.join(" "), /resolves to SDPA/); +}); + +test("Cache-DiT changes only the request and never inherits uncached verification", () => { + const s = selection(generation, { workload: "smoke" }); + assert.equal(generation.commandBuilder.resolveDeployment(s).builder.verification.request, "verified"); + for (const mode of ["text", "image"]) { + const uncached = { ...s, mode }; + const cached = { ...uncached, cache: "on" }; + assert.deepEqual(flags(generation, cached), flags(generation, uncached)); + assert.match(generation.curl(cached), mode === "text" ? /"enable_cache_dit": true/ : /enable_cache_dit=true/); + assert.match(generation.curl(uncached), mode === "text" ? /"enable_cache_dit": false/ : /enable_cache_dit=false/); + const builder = generation.commandBuilder.resolveDeployment(cached).builder; + assert.equal(builder.verification.serve, "verified"); + assert.equal(builder.verification.request, "unverified"); + assert.match(builder.warnings.join(" "), /Cache-DiT is approximate/); + } + const fsdp = generation.commandBuilder.resolveDeployment(selection(generation, { + hw: "gb300", gpus_per_node: 4, placement: "fsdp", cache: "on", + })).builder; + assert.equal(fsdp.errors.length, 0); +}); + +test("SR FSDP is selectable on multiple GPUs without claiming E2E verification", () => { + const s = selection(sr, { placement: "fsdp" }); + const builder = sr.commandBuilder.resolveDeployment(s).builder; + assert.ok(flags(sr, s).includes("--use-fsdp-inference true")); + assert.equal(builder.errors.length, 0); + assert.equal(builder.verification.serve, "unverified"); + assert.equal(sr.commandBuilder.resolveDeployment({ ...s, gpus_per_node: 1 }).builder.verification.serve, "error"); +}); + +test("SR non-default attention and layer counts do not inherit verified HTTP recipes", () => { + const s = selection(sr, { + placement: "offload", topology_mode: "manual", gpus_per_node: 2, + tp_size: 1, ulysses_degree: 1, ring_degree: 2, + }); + assert.equal(sr.commandBuilder.resolveDeployment(s).builder.verification.serve, "verified"); + for (const overrides of [{ prefetch: 2 }, { resident_layers: 1 }, { offload_components: "transformer" }]) { + assert.equal(sr.commandBuilder.resolveDeployment({ ...s, ...overrides }).builder.verification.serve, "unverified"); + } + const sdpa = sr.commandBuilder.resolveDeployment(selection(sr, { + attention: "torch_sdpa", topology_mode: "manual", tp_size: 2, ulysses_degree: 2, + })).builder; + assert.equal(sdpa.errors.length, 0); + assert.equal(sdpa.verification.serve, "unverified"); +}); + +test("SR decode modes emit mutually exclusive flags and explicit numerical boundaries", () => { + for (const decode of ["serial", "tiles", "spatial"]) { + const s = selection(sr, { decode }); + const command = flags(sr, s); + assert.ok(command.includes(`--vae-config.use-parallel-tiling ${decode === "tiles"}`)); + assert.ok(command.includes(`--vae-config.use-parallel-decode ${decode === "spatial"}`)); + if (decode === "spatial") { + assert.ok(command.includes("--vae-config.parallel-decode-mode spatial_shard")); + const warnings = sr.commandBuilder.resolveDeployment(s).builder.warnings.join(" "); + assert.match(warnings, /Not bitwise lossless/); + assert.match(warnings, /57.51/); + assert.match(warnings, /4\/255/); + const option = sr.overlayDims.find((dim) => dim.id === "decode").options.find((entry) => entry.id === decode); + assert.match(option.hints.join(" "), /Not bitwise lossless/); + } + } +}); + +test("SR topology uses the released checkpoint's 28 heads, not the class default", () => { + const resource = sr.commandBuilder.resource; + assert.ok(resource.validateTopology({ nodes: 1, gpus_per_node: 8 }, { + tp_size: 1, ulysses_degree: 8, ring_degree: 1, + }).some((message) => message.includes("28 DiT heads"))); +}); + +test("SR hardware extrapolation never inherits GB300 execution verification", () => { + for (const hw of ["b200", "h200", "h100", "rtx5090"]) { + assert.ok(sr.supportedHardware.includes(hw)); + for (const gpus_per_node of [1, 2, 4]) { + const s = selection(sr, { hw, gpus_per_node }); + const builder = sr.commandBuilder.resolveDeployment(s).builder; + assert.equal(builder.errors.length, 0); + assert.equal(builder.verification.serve, "unverified"); + assert.equal(builder.verification.request, "unverified"); + assert.match(builder.warnings.join(" "), /Extrapolated hardware recipe/); + assert.ok(flags(sr, s).includes(`--attention-backend ${hw === "rtx5090" ? "torch_sdpa" : "fa"}`)); + } + } + const consumer = sr.commandBuilder.resource.verifiedRecipes.find((r) => r.hw === "rtx5090"); + assert.equal(consumer.gpus_per_node, 1); + assert.equal(consumer.placement, "offload"); + assert.equal(consumer.unverified, true); + assert.match(sr.commandBuilder.resolveDeployment(selection(sr, { hw: "rtx5090" })) + .builder.warnings.join(" "), /may still OOM/); +}); + +test("SR eight-GPU auto topology respects heads and backend Ring capability", () => { + assert.equal(sr.commandBuilder.resource.limits.gpus_per_node.max, 8); + for (const hw of ["gb300", "b200", "h200", "h100"]) { + const s = selection(sr, { hw, gpus_per_node: 8 }); + const builder = sr.commandBuilder.resolveDeployment(s).builder; + assert.deepEqual(builder.topology, { tp_size: 1, ulysses_degree: 4, ring_degree: 2 }); + assert.equal(builder.errors.length, 0); + assert.equal(builder.verification.serve, "unverified"); + assert.match(builder.warnings.join(" "), /Eight GPUs.*Ulysses4 x Ring2/); + } + for (const hw of ["rtxpro6000", "rtx5090"]) { + const builder = sr.commandBuilder.resolveDeployment(selection(sr, { hw, gpus_per_node: 8 })).builder; + assert.equal(builder.verification.serve, "error"); + assert.match(builder.errors.join(" "), /SDPA.*Ring/); + assert.match(sr.commandBuilder.resolveDeployment(selection(sr, { hw, attention: "fa" })) + .builder.errors.join(" "), /resolves to SDPA/); + } +}); + +test("SR RTX HTTP verification covers only the executed placements and topologies", () => { + for (const [gpus_per_node, decode] of [[1, "serial"], [2, "tiles"]]) { + const s = selection(sr, { hw: "rtxpro6000", gpus_per_node, decode }); + assert.equal(sr.commandBuilder.resolveDeployment(s).builder.verification.serve, "verified"); + assert.equal(sr.commandBuilder.resolveDeployment(s).builder.verification.request, "unverified"); + for (const overrides of [{ placement: "offload" }, { decode: "spatial" }, { gpus_per_node: 4 }]) { + assert.equal(sr.commandBuilder.resolveDeployment({ ...s, ...overrides }).builder.verification.serve, "unverified"); + } + } + const recipe = sr.commandBuilder.resource.verifiedRecipes.find((r) => r.hw === "rtxpro6000" && r.default); + assert.equal(recipe.decode, "serial"); + assert.equal(recipe.unverified, false); +}); + +test("SR tile batching changes only the request and warns about changed output", () => { + const s = selection(sr); + const batched = { ...s, tile_batch: "2", scale: "2.25" }; + assert.deepEqual(flags(sr, s), flags(sr, batched)); + assert.match(sr.curl(batched), /sr_tiles_batch_size=2/); + assert.match(sr.curl(batched), /sr_resolution_scale=2.25/); + assert.match(sr.curl(batched), /video_reference=@/); + assert.match(sr.commandBuilder.resolveDeployment(batched).builder.warnings.join(" "), /not lossless/); +}); + +test("SR execution verification does not promise quality or arbitrary-video verification", () => { + const s = selection(sr, { topology_mode: "manual", tp_size: 2, ulysses_degree: 2 }); + const builder = sr.commandBuilder.resolveDeployment(s).builder; + assert.equal(builder.verification.serve, "verified"); + assert.equal(builder.verification.request, "unverified"); + assert.match(builder.warnings.join(" "), /not lossless output/); + assert.match(sr.commandBuilder.resolveDeployment(selection(sr, { gpus_per_node: 1 })).builder.warnings.join(" "), /no parallel speedup/); +}); + +test("both cookbook pages use the shared builder and consistent ports", () => { + for (const name of ["Kandinsky6", "Kandinsky6-SR"]) { + const page = read(`cookbook/diffusion/Kandinsky/${name}.mdx`); + assert.match(page, //); + assert.ok(page.indexOf("## 1. Quick start") < page.indexOf("## 2. Model capabilities")); + assert.match(page, /HF_TOKEN/); + } + for (const config of [generation, sr]) { + assert.equal(config.placeholders.PORT.default, "30000"); + assert.equal(config.placeholders.CURL_PORT.default, "30000"); + } + const page = read("cookbook/diffusion/Kandinsky/Kandinsky6-SR.mdx"); + assert.match(page, /Spatial shard is not bitwise lossless/); + assert.match(page, /57.51 dB/); + assert.match(page, /0.99921/); + assert.match(page, /4\/255/); +}); diff --git a/docs/src/snippets/configs/Kandinsky/kandinsky6-sr.jsx b/docs/src/snippets/configs/Kandinsky/kandinsky6-sr.jsx new file mode 100644 index 0000000000..b1f24aa50f --- /dev/null +++ b/docs/src/snippets/configs/Kandinsky/kandinsky6-sr.jsx @@ -0,0 +1,245 @@ +export const config = { + modelName: "Kandinsky 6.0 SR", + supportedHardware: ["gb300", "b200", "h200", "h100", "rtxpro6000", "rtx5090"], + hardware: [ + { id: "rtxpro6000", label: "RTX PRO 6000", vram: "96GB", vendor: "blackwell" }, + { id: "rtx5090", label: "RTX 5090", vram: "32GB", vendor: "blackwell" }, + ], + groupHardware: false, + matchDims: [], + overlayDims: [ + { + id: "attention", title: "Attention", scope: "serve", default: "platform", + description: "Dense attention. Switching kernels can change rounding; neither choice enables NABLA.", + learnMore: "#5-memory-and-parallelism", + options: [ + { id: "platform", label: "Recipe default", recommended: true, + description: "SDPA on RTX; FA on datacenter Blackwell and Hopper." }, + { id: "fa", label: "FlashAttention", + disabled: (s) => ["rtxpro6000", "rtx5090"].includes(s.hw), + disableReason: "This SM120 platform maps FA to SDPA; select SDPA explicitly instead." }, + { id: "torch_sdpa", label: "PyTorch SDPA", + soft: (s) => s.hw !== "rtxpro6000", + softReason: "SDPA measurements apply to RTX PRO 6000, not every GPU or workload.", + disabled: (s) => s.topology_mode === "manual" && Number(s.ring_degree) > 1, + disableReason: "SDPA does not support Ring rotation." }, + ], + }, + { + id: "decode", title: "VAE decode", scope: "serve", default: "tiles", + description: "Whole-tile parallelism preserves the tested serial output; spatial sharding is not bitwise lossless.", + learnMore: "#4-decode-speed-memory-and-numerics", + options: [ + { id: "serial", label: "Serial", + description: "Reference decode. Each rank processes all tiles; single-GPU and single-tile baseline.", + flags: ["--vae-config.use-parallel-tiling false", "--vae-config.use-parallel-decode false"] }, + { id: "tiles", label: "Whole-tile parallel", recommended: true, + description: "Distributes complete tiles. Tested RGB/audio are exact versus serial on the same topology. Single-tile inputs fall back to serial.", + hints: ["Whole-tile decode matched serial RGB/audio in tested same-topology runs."], + flags: ["--vae-config.use-parallel-tiling true", "--vae-config.use-parallel-decode false"] }, + { id: "spatial", label: "Spatial shard (rounding changes)", + description: "Lower activation memory, not bitwise lossless. GB300 measurement: PSNR 57.51 dB, min sampled SSIM 0.99921, max pixel difference 4/255; audio exact.", + hints: ["Not bitwise lossless: spatial decode changes rounding; GB300 PSNR 57.51 dB, max pixel difference 4/255."], + flags: ["--vae-config.use-parallel-tiling false", "--vae-config.use-parallel-decode true", "--vae-config.parallel-decode-mode spatial_shard"] }, + ], + }, + { + id: "placement", title: "Weight placement", scope: "serve", default: "resident", + description: "Offload reduces GPU weight memory, but adds host memory and transfers.", + learnMore: "#5-memory-and-parallelism", + options: [ + { id: "resident", label: "Resident", recommended: true }, + { id: "component", label: "Component offload", soft: true, + description: "Transfers each selected component whole around its use. This exact SR recipe is unverified.", + flags: (s) => [`--cpu-offload-components ${s.offload_components}`] }, + { id: "offload", label: "Layerwise offload", + flags: (s) => [`--layerwise-offload-components ${s.offload_components}`], + description: "Streams supported DiT, KVAE and latent-upscaler blocks. It can be slower than resident execution." }, + { id: "fsdp", label: "FSDP", soft: true, + softReason: "SR weight sharding has not completed full-checkpoint E2E validation.", + description: "Shards DiT weights across GPUs. Selectable for validation, not a verified memory or latency recommendation.", + flags: ["--use-fsdp-inference true"], + disabled: (s) => Number(s.gpus_per_node) < 2, + disableReason: "Weight sharding requires multiple GPUs." }, + ], + }, + { + id: "offload_components", title: "Offload components", scope: "serve", default: "all", + showWhen: (s) => ["component", "offload"].includes(s.placement), + description: "Choose which weights leave GPU memory. Partial selections are unverified recipes.", + learnMore: "#5-memory-and-parallelism", + options: [ + { id: "all", label: "All components" }, + { id: "transformer", label: "DiT", soft: true }, + { id: "vae", label: "VAE", soft: true }, + { id: "latent_upscaler", label: "Latent upscaler", soft: true }, + { id: "transformer vae", label: "DiT + VAE", soft: true }, + { id: "transformer latent_upscaler", label: "DiT + upscaler", soft: true }, + { id: "vae latent_upscaler", label: "VAE + upscaler", soft: true }, + ], + }, + { + id: "prefetch", title: "DiT prefetch", scope: "serve", kind: "number", + min: 1, max: 4, default: 1, unit: "layers", options: [], + showWhen: (s) => s.placement === "offload" && (s.offload_components === "all" || s.offload_components.includes("transformer")), + verifiedWhen: (s) => Number(s.prefetch) === 1, + description: "More prefetched layers use more VRAM and may overlap transfers better. Values above 1 are unverified here.", + learnMore: "#5-memory-and-parallelism", + }, + { + id: "resident_layers", title: "DiT resident layers", scope: "serve", kind: "number", + min: 0, max: 8, default: 0, unit: "layers", options: [], + showWhen: (s) => s.placement === "offload" && (s.offload_components === "all" || s.offload_components.includes("transformer")), + verifiedWhen: (s) => Number(s.resident_layers) === 0, + description: "Keep leading DiT layers on GPU during the request. Trades VRAM for fewer transfers; nonzero recipes are unverified.", + learnMore: "#5-memory-and-parallelism", + }, + { + id: "scale", title: "Upscale factor", scope: "request", default: "2", + description: "Choose output scale, not quality. All use the two-step distilled checkpoint.", + learnMore: "#request-parameters", + options: [{ id: "2", label: "2x" }, { id: "4", label: "4x" }, { id: "2.25", label: "2.25x" }], + }, + { + id: "tile_batch", title: "Tile batch", scope: "request", default: "1", + description: "Changing tile batching changes seeded noise and therefore the output.", + options: [ + { id: "1", label: "1", recommended: true }, + { id: "2", label: "2 (different output)", + description: "Uses more activation memory. Not an output-preserving acceleration." }, + ], + }, + ], + commandBuilder: { + defaultSelection: { + hw: "gb300", nodes: 1, gpus_per_node: 4, topology_mode: "auto", + tp_size: 1, ulysses_degree: 4, ring_degree: 1, + }, + resource: { + limits: { nodes: { min: 1, max: 1 }, gpus_per_node: { min: 1, max: 8 } }, + verifiedRecipes: [ + { id: "gb300-ulysses4-offline", hw: "gb300", nodes: 1, gpus_per_node: 4, + tp_size: 1, ulysses_degree: 4, ring_degree: 1, placement: "resident", + decode: "tiles", default: true, unverified: true }, + ...["b200", "h200", "h100", "rtxpro6000", "rtx5090"].map((hw) => ({ + id: `${hw}-single-gpu`, hw, nodes: 1, gpus_per_node: 1, + tp_size: 1, ulysses_degree: 1, ring_degree: 1, + placement: hw === "rtx5090" ? "offload" : "resident", + decode: "serial", default: true, unverified: hw !== "rtxpro6000", + })), + { id: "rtxpro6000-ulysses2-tiles", hw: "rtxpro6000", nodes: 1, gpus_per_node: 2, + tp_size: 1, ulysses_degree: 2, ring_degree: 1, placement: "resident", decode: "tiles" }, + ], + autoTopology: (s) => ({ + tp_size: 1, + ulysses_degree: Number(s.gpus_per_node) === 8 ? 4 : Number(s.gpus_per_node), + ring_degree: Number(s.gpus_per_node) === 8 ? 2 : 1, + }), + validateTopology: (s, t) => { + const errors = []; + const degrees = [t.tp_size, t.ulysses_degree, t.ring_degree]; + if (!degrees.every((n) => Number.isInteger(n) && n >= 1)) + errors.push("Parallel degrees must be positive integers."); + if (Number(s.nodes) !== 1) errors.push("This picker covers one node."); + if (degrees.reduce((a, b) => a * b, 1) !== Number(s.gpus_per_node)) + errors.push("GPU count must equal TP times Ulysses times Ring."); + if (28 % (t.tp_size * t.ulysses_degree)) + errors.push("TP times Ulysses must divide the 28 DiT heads."); + if (s.placement === "fsdp" && Number(s.gpus_per_node) < 2) + errors.push("FSDP requires multiple GPUs."); + if ((s.attention === "torch_sdpa" || ["rtxpro6000", "rtx5090"].includes(s.hw)) && t.ring_degree > 1) + errors.push("SDPA does not support Ring rotation."); + if (s.attention === "fa" && ["rtxpro6000", "rtx5090"].includes(s.hw)) + errors.push("FA resolves to SDPA on this SM120 platform; select SDPA instead."); + return errors; + }, + }, + resolveDeployment: (s) => { + const resource = config.commandBuilder.resource; + const topology = s.topology_mode === "manual" + ? { tp_size: Number(s.tp_size), ulysses_degree: Number(s.ulysses_degree), ring_degree: Number(s.ring_degree) } + : resource.autoTopology(s); + const errors = resource.validateTopology(s, topology); + const platformAttention = ["rtxpro6000", "rtx5090"].includes(s.hw) ? "torch_sdpa" : "fa"; + const attention = s.attention === "platform" ? platformAttention : s.attention; + const streamingDit = s.placement === "offload" && (s.offload_components === "all" || s.offload_components.includes("transformer")); + const gb300Served = s.hw === "gb300" && attention === "fa" && ( + (topology.tp_size === 2 && topology.ulysses_degree === 2 && topology.ring_degree === 1 + && s.placement === "resident" && s.decode !== "spatial") + || (topology.tp_size === 1 && topology.ulysses_degree === 1 && topology.ring_degree === 2 + && s.placement === "offload" && s.offload_components === "all" + && Number(s.prefetch) === 1 && Number(s.resident_layers) === 0 && s.decode !== "spatial") + ); + const rtxServed = s.hw === "rtxpro6000" && attention === "torch_sdpa" + && s.placement === "resident" && topology.tp_size === 1 && topology.ring_degree === 1 + && ((topology.ulysses_degree === 1 && s.decode === "serial") + || (topology.ulysses_degree === 2 && s.decode === "tiles")); + const served = errors.length === 0 && (gb300Served || rtxServed); + const warnings = [ + "Verification describes execution, not lossless output. Arbitrary uploaded videos are outside the fixed HTTP test workload.", + ]; + if (!served) warnings.push("This exact HTTP recipe is unverified. Offline timing does not verify every server and uploaded-video combination."); + if (!["gb300", "rtxpro6000"].includes(s.hw)) + warnings.push("Extrapolated hardware recipe: no SR latency or peak-memory measurement on this GPU. Validate a short clip before scaling up."); + if (attention !== platformAttention) + warnings.push("This backend override is a different numerical path; its full-checkpoint output and performance are unverified."); + if (s.hw === "rtx5090") + warnings.push("32GB VRAM: start with layerwise offload, tile batch 1 and a short 384 x 256 source. Offload does not bound activation memory; full-size video may still OOM."); + if (Number(s.gpus_per_node) > 4) + warnings.push("More than four GPUs is extrapolated. Eight GPUs use Ulysses4 x Ring2 with FA; Ulysses8 cannot divide 28 heads."); + if (s.hw === "rtxpro6000" && Number(s.gpus_per_node) > 2) + warnings.push("RTX PRO 6000 measurements cover one or two GPUs only; larger GPU counts are extrapolated."); + if (["component", "offload"].includes(s.placement)) + warnings.push("Offload uses host RAM and transfer bandwidth; prefetch and resident layers additionally consume VRAM."); + if (s.decode === "spatial") warnings.push("Not bitwise lossless: spatial decode changes rounding. Tested PSNR 57.51 dB, max pixel difference 4/255; validate your video."); + if (s.decode === "tiles") warnings.push("Whole-tile decode matched serial RGB/audio exactly on tested workloads within each topology, not across different TP/Ring settings."); + if (topology.tp_size > 1 || topology.ring_degree > 1) + warnings.push("TP/Ring can change output numerics independently of the VAE decode mode."); + if (s.tile_batch !== "1") warnings.push("Tile batch 2 changes seeded noise and output; it is not lossless."); + if (Number(s.gpus_per_node) === 1 && s.decode !== "serial") + warnings.push("One GPU uses serial decoding; this selection provides no parallel speedup."); + return { + match: { hw: s.hw }, nnodes: Number(s.nodes), verified: false, + flags: [ + "--model-path {{MODEL_NAME}}", `--num-gpus ${Number(s.gpus_per_node)}`, + `--tp-size ${topology.tp_size}`, `--ulysses-degree ${topology.ulysses_degree}`, + `--ring-degree ${topology.ring_degree}`, "--performance-mode speed", + `--attention-backend ${attention}`, "--warmup-mode off", "--host {{HOST_IP}}", "--port {{PORT}}", + ...(streamingDit && Number(s.prefetch) !== 1 ? [`--dit-offload-prefetch-size ${s.prefetch}`] : []), + ...(streamingDit && Number(s.resident_layers) !== 0 ? [`--dit-layerwise-resident-layers ${s.resident_layers}`] : []), + ], + builder: { + topology, + resolvedSettings: { + attention: attention === "fa" ? "FlashAttention" : "PyTorch SDPA", + prefetch: `${s.prefetch} ${Number(s.prefetch) === 1 ? "layer" : "layers"}`, + resident_layers: `${s.resident_layers} ${Number(s.resident_layers) === 1 ? "layer" : "layers"}`, + }, + topologySummary: `TP ${topology.tp_size}, Ulysses ${topology.ulysses_degree}, Ring ${topology.ring_degree}`, + errors, warnings, + verification: { + serve: errors.length ? "error" : served ? "verified" : "unverified", + request: errors.length ? "error" : "unverified", + }, + }, + }; + }, + }, + modelNames: { default: "kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers" }, + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + INPUT_VIDEO: { target: "curl", label: "Source video", default: "/path/to/input.mp4" }, + }, + curl: (s) => `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/videos \\ + -F "video_reference=@{{INPUT_VIDEO}};type=video/mp4" \\ + -F sr_resolution_scale=${s.scale} \\ + -F sr_tiles_batch_size=${s.tile_batch} \\ + -F num_inference_steps=2 \\ + -F seed=42`, + runModes: () => ["python"], + showPlaygroundLink: false, + cells: [], +}; diff --git a/docs/src/snippets/configs/Kandinsky/kandinsky6.jsx b/docs/src/snippets/configs/Kandinsky/kandinsky6.jsx new file mode 100644 index 0000000000..86d4dbab67 --- /dev/null +++ b/docs/src/snippets/configs/Kandinsky/kandinsky6.jsx @@ -0,0 +1,325 @@ +export const config = { + modelName: "Kandinsky 6.0", + supportedHardware: ["b200", "gb300", "rtxpro6000"], + hardware: [ + { id: "rtxpro6000", label: "RTX PRO 6000", vram: "96GB", vendor: "blackwell" }, + ], + groupHardware: false, + matchDims: [], + overlayDims: [ + { + id: "weights", + title: "Checkpoint", + scope: "base", + default: "distill", + description: + "Pro-distill uses 10 steps without CFG; Pro uses 50 steps with CFG.", + options: [ + { id: "distill", label: "Pro-distill", recommended: true }, + { id: "pro", label: "Pro" }, + ], + }, + { + id: "mode", + title: "Conditioning", + scope: "base", + default: "text", + description: + "Both modes generate video and audio through the same server.", + options: [ + { id: "text", label: "Text" }, + { id: "image", label: "Text and image" }, + ], + }, + { + id: "attention", title: "Attention", scope: "serve", default: "platform", + description: "The recipe uses FA on B200/GB300 and SDPA on RTX PRO 6000. Sage3 overrides only the DiT; kernel changes can alter both outputs.", + learnMore: "#rtx-pro-6000-and-sageattention3", + options: [ + { id: "platform", label: "Recipe default", recommended: true }, + { id: "fa", label: "FlashAttention", + disabled: (s) => s.hw === "rtxpro6000", + disableReason: "This SM120 platform maps FA to SDPA; select SDPA explicitly instead." }, + { id: "torch_sdpa", label: "PyTorch SDPA", + soft: (s) => s.hw !== "rtxpro6000", + softReason: "Full-checkpoint SDPA validation used RTX PRO 6000.", + disabled: (s) => s.topology_mode === "manual" && Number(s.ring_degree) > 1, + disableReason: "SDPA does not support Ring rotation." }, + { id: "sage_attn_3", label: "Sage3 (approximate)", soft: true, + disabled: (s) => s.hw !== "rtxpro6000", + disableReason: "The tested Sage3 build requires SM120/121; it rejects B200/GB300.", + softReason: "Offline benchmark only; this HTTP recipe is unverified. Requires a separate SageAttention3 installation.", + description: "RTX: 21.3% lower latency, but video PSNR 11.97 dB and audio SNR -2.21 dB versus SDPA. Not lossless or quality-certified.", + hints: ["Sage3 is approximate and changes video/audio substantially. Install the documented SM120 SageAttention3 build first."] }, + ], + }, + { + id: "placement", + title: "Weight placement", + scope: "serve", + default: "resident", + description: "Keep weights resident when they fit; trade transfer time for lower GPU memory otherwise.", + learnMore: "#choosing-a-topology", + options: [ + { id: "resident", label: "Resident", recommended: true, + description: "Baseline arithmetic, without weight transfers between stages." }, + { id: "fsdp", label: "FSDP", + description: "Shards DiT weights. GB300 Ulysses4 matched resident RGB/audio, using less GPU memory at higher latency.", + flags: ["--use-fsdp-inference true"], + disabled: (s) => Number(s.gpus_per_node) < 2, + disableReason: "Weight sharding requires multiple GPUs." }, + { id: "component", label: "Component offload", soft: true, + description: "Transfers selected components whole around their use. Partial recipes need their own memory and latency validation.", + flags: (s) => [`--cpu-offload-components ${s.offload_components}`] }, + { id: "offload", label: "Layerwise offload", + description: "Streams blocks from host RAM. This adds transfers and pinned host memory; not a free speedup.", + flags: (s) => [`--layerwise-offload-components ${s.offload_components}`] }, + ], + }, + { + id: "offload_components", title: "Offload components", scope: "serve", default: "all", + showWhen: (s) => ["component", "offload"].includes(s.placement), + description: "Text encoders includes both encoders; VAEs includes video and audio. Partial selections are unverified recipes.", + learnMore: "#choosing-a-topology", + options: [ + { id: "all", label: "All components" }, + { id: "transformer", label: "DiT", soft: true }, + { id: "text_encoder", label: "Text encoders", soft: true }, + { id: "vae audio_vae", label: "VAEs", soft: true }, + { id: "audio_vae", label: "Audio VAE", soft: true }, + { id: "text_encoder vae audio_vae", label: "Encoders + VAEs", soft: true }, + ], + }, + { + id: "prefetch", title: "DiT prefetch", scope: "serve", kind: "number", + min: 1, max: 4, default: 1, unit: "layers", options: [], + showWhen: (s) => s.placement === "offload" && ["all", "transformer"].includes(s.offload_components), + verifiedWhen: (s) => Number(s.prefetch) === 1, + description: "More prefetched layers use more VRAM and may overlap transfers better. Values above 1 are unverified here.", + learnMore: "#choosing-a-topology", + }, + { + id: "resident_layers", title: "DiT resident layers", scope: "serve", kind: "number", + min: 0, max: 8, default: 0, unit: "layers", options: [], + showWhen: (s) => s.placement === "offload" && ["all", "transformer"].includes(s.offload_components), + verifiedWhen: (s) => Number(s.resident_layers) === 0, + description: "Keep leading DiT layers on GPU during the request. Trades VRAM for fewer transfers; nonzero recipes are unverified.", + learnMore: "#choosing-a-topology", + }, + { + id: "workload", + title: "Workload", + scope: "request", + default: "full", + description: + "Use the lightweight request to check deployment, not output quality.", + options: [ + { id: "full", label: "768 x 512, 121 frames", recommended: true }, + { id: "smoke", label: "384 x 256, 17 frames, 2 steps" }, + ], + }, + { + id: "cache", title: "Cache-DiT", scope: "request", default: "off", + description: "Request-level block reuse can change both video and audio. The default cache policy is not a validated quality preset.", + learnMore: "#cache-dit", + options: [ + { id: "off", label: "Off", recommended: true }, + { id: "on", label: "On (approximate)", soft: true, + description: "Uses the runtime cache defaults. Short requests may finish before cache warmup ends, with no acceleration." }, + ], + }, + ], + commandBuilder: { + defaultSelection: { + hw: "b200", + nodes: 1, + gpus_per_node: 1, + topology_mode: "auto", + tp_size: 1, + ulysses_degree: 1, + ring_degree: 1, + }, + resource: { + limits: { nodes: { min: 1, max: 1 }, gpus_per_node: { min: 1, max: 4 } }, + verifiedRecipes: [ + ["b200", 1, 1], + ["b200", 2, 1], + ["gb300", 4, 1], + ["gb300", 4, 2], + ["rtxpro6000", 1, 1], + ].map(([hw, gpus, tp]) => ({ + id: `${hw}-distill-${gpus}-tp${tp}`, + hw, + weights: "distill", + placement: "resident", + nodes: 1, + gpus_per_node: gpus, + tp_size: tp, + ulysses_degree: gpus / tp, + ring_degree: 1, + default: tp === 1 && (hw === "gb300" ? gpus === 4 : gpus === 1), + })).concat([{ + id: "gb300-distill-fsdp", + hw: "gb300", weights: "distill", placement: "fsdp", + nodes: 1, gpus_per_node: 4, tp_size: 1, ulysses_degree: 4, ring_degree: 1, + }]), + autoTopology: (s) => ({ + tp_size: 1, + ulysses_degree: Number(s.gpus_per_node), + ring_degree: 1, + }), + validateTopology: (s, t) => { + const errors = []; + const degrees = [t.tp_size, t.ulysses_degree, t.ring_degree]; + if (!degrees.every((n) => Number.isInteger(n) && n >= 1)) + errors.push("Parallel degrees must be positive integers."); + if (Number(s.nodes) !== 1) errors.push("This picker covers one node."); + if (degrees.reduce((a, b) => a * b, 1) !== Number(s.gpus_per_node)) + errors.push("GPU count must equal TP times Ulysses times Ring."); + if (32 % (t.tp_size * t.ulysses_degree)) + errors.push("TP times Ulysses must divide the 32 DiT heads."); + if (s.placement === "fsdp" && Number(s.gpus_per_node) < 2) + errors.push("FSDP requires multiple GPUs."); + if ((["torch_sdpa", "sage_attn_3"].includes(s.attention) || s.hw === "rtxpro6000") && t.ring_degree > 1) + errors.push("SDPA and Sage3 do not support Ring rotation; use FlashAttention on B200/GB300."); + if (s.attention === "fa" && s.hw === "rtxpro6000") + errors.push("FA resolves to SDPA on this SM120 platform; select SDPA instead."); + if (s.attention === "sage_attn_3" && s.hw !== "rtxpro6000") + errors.push("The tested Sage3 build requires SM120/121."); + return errors; + }, + }, + resolveDeployment: (s) => { + const resource = config.commandBuilder.resource; + const topology = + s.topology_mode === "manual" + ? { + tp_size: Number(s.tp_size), + ulysses_degree: Number(s.ulysses_degree), + ring_degree: Number(s.ring_degree), + } + : resource.autoTopology(s); + const errors = resource.validateTopology(s, topology); + const platformAttention = s.hw === "rtxpro6000" ? "torch_sdpa" : "fa"; + const attention = s.attention === "platform" ? platformAttention : s.attention; + const streamingDit = s.placement === "offload" && ["all", "transformer"].includes(s.offload_components); + const recipe = resource.verifiedRecipes.find( + (r) => + r.hw === s.hw && + r.weights === s.weights && + r.placement === s.placement && + r.nodes === Number(s.nodes) && + r.gpus_per_node === Number(s.gpus_per_node) && + r.tp_size === topology.tp_size && + r.ulysses_degree === topology.ulysses_degree && + r.ring_degree === topology.ring_degree, + ); + const served = !!recipe && errors.length === 0 && attention === platformAttention; + const requested = served && s.workload === "smoke" && s.cache === "off"; + const warnings = []; + if (!requested) + warnings.push( + served + ? "Only the uncached lightweight workload is verified for this HTTP recipe; full-size generation is checked offline." + : "This exact checkpoint, placement and topology combination has not completed HTTP verification.", + ); + if (topology.tp_size > 1) + warnings.push( + "TP changes video and audio outputs. Prefer Ulysses when memory permits; validate quality for your workload.", + ); + if (topology.ring_degree > 1) + warnings.push("Ring changes floating-point reductions; bitwise video/audio parity is not established."); + if (["component", "offload"].includes(s.placement)) + warnings.push("Offload needs additional host RAM and transfer bandwidth; DiT prefetch and resident layers also consume VRAM."); + if (attention !== platformAttention) + warnings.push("This attention override has not completed the exact HTTP recipe; changing kernels can change video and audio."); + if (attention === "sage_attn_3") + warnings.push("Sage3 is approximate: measured video PSNR 11.97 dB and audio SNR -2.21 dB versus SDPA; no quality guarantee."); + if (s.cache === "on") + warnings.push("Cache-DiT is approximate and affects both streams; validate video and audio. Short requests may not skip any blocks."); + const model = + s.weights === "pro" + ? "kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers" + : "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"; + return { + match: { hw: s.hw }, + nnodes: Number(s.nodes), + verified: requested, + flags: [ + `--model-path ${model}`, + `--num-gpus ${Number(s.gpus_per_node)}`, + `--tp-size ${topology.tp_size}`, + `--ulysses-degree ${topology.ulysses_degree}`, + `--ring-degree ${topology.ring_degree}`, + "--performance-mode speed", + `--attention-backend ${attention === "sage_attn_3" ? platformAttention : attention}`, + ...(attention === "sage_attn_3" ? ["--component-attention-backends transformer=sage_attn_3"] : []), + "--warmup-mode off", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ...(streamingDit && Number(s.prefetch) !== 1 ? [`--dit-offload-prefetch-size ${s.prefetch}`] : []), + ...(streamingDit && Number(s.resident_layers) !== 0 ? [`--dit-layerwise-resident-layers ${s.resident_layers}`] : []), + ], + builder: { + topology, + resolvedSettings: { + attention: { fa: "FlashAttention", torch_sdpa: "PyTorch SDPA", sage_attn_3: "Sage3 (approximate)" }[attention], + prefetch: `${s.prefetch} ${Number(s.prefetch) === 1 ? "layer" : "layers"}`, + resident_layers: `${s.resident_layers} ${Number(s.resident_layers) === 1 ? "layer" : "layers"}`, + }, + topologySummary: `TP ${topology.tp_size}, Ulysses ${topology.ulysses_degree}, Ring ${topology.ring_degree}`, + errors, + warnings, + verification: { + serve: errors.length ? "error" : served ? "verified" : "unverified", + request: errors.length + ? "error" + : requested + ? "verified" + : "unverified", + }, + }, + }; + }, + }, + modelNames: { + default: "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers", + }, + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + INPUT_IMAGE: { + target: "curl", + label: "Reference PNG", + default: "/path/to/reference.png", + }, + }, + curl: (s) => { + const smoke = s.workload === "smoke"; + const body = { + prompt: + "A woman chops vegetables in a sunlit kitchen, with soft jazz playing.", + size: smoke ? "384x256" : "768x512", + num_frames: smoke ? 17 : 121, + fps: 24, + seed: 42, + enable_cache_dit: s.cache === "on", + }; + if (smoke) body.num_inference_steps = 2; + if (s.mode === "text") + return `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/videos \\ + -H 'Content-Type: application/json' \\ + -d '${JSON.stringify(body, null, 2)}'`; + const fields = Object.entries(body).map( + ([key, value]) => ` --form-string '${key}=${value}'`, + ); + fields.push(' -F "input_reference=@{{INPUT_IMAGE}};type=image/png"'); + return `curl -sS --fail-with-body http://{{CURL_HOST}}:{{CURL_PORT}}/v1/videos \\ +${fields.join(" \\\n")}`; + }, + runModes: () => ["python"], + showPlaygroundLink: false, + cells: [], +}; diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx index fea6a805b7..0a4dc67152 100644 --- a/docs/src/snippets/diffusion/model-catalog.jsx +++ b/docs/src/snippets/diffusion/model-catalog.jsx @@ -134,6 +134,24 @@ export const DiffusionModelCatalog = ({ category }) => { }, ], video: [ + { + name: "Kandinsky 6", + modelIds: [ + "kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers", + "kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers", + "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers", + ], + cookbook: "/cookbook/diffusion/Kandinsky/Kandinsky6", + note: "Pro and Pro-distill have full-checkpoint validation; the older Pro-sft entry remains unverified.", + }, + { + name: "Kandinsky 6 VSR", + modelIds: [ + "kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", + ], + cookbook: "/cookbook/diffusion/Kandinsky/Kandinsky6-SR", + note: "Use VSR-distilled2steps. The non-distilled VSR output head is incompatible with the current Euler path.", + }, { name: "Wan 2.1", modelIds: [ diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index 24722c37d8..ca656afefe 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -151,6 +151,10 @@ Several norms look interchangeable and are not. Start here. | Entry point | Backend | Contract | Applies to | |---|---|---|---| | `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) | +| `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 | + +These contracts differ: `residual_gate_fp32` does not round the product to +BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary. The transposed-dense path tiles along the physical stride-1 token dimension for coalesced residual reads and output writes. The contiguous layouts use diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 5319dc79ae..d1d2a869dc 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -49,6 +49,20 @@ _HIP = frozenset({CapabilityRequirement.HIP}) # inventory: callers name the one they want via ``select_kernel``. # --------------------------------------------------------------------------- _SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( + ( + "diffusion.residual_gate_fp32", + KernelBackend.TRITON, + "modulate.residual_gate_fp32:residual_gate_fp32", + _CUDA, + "Residual gating with separate FP32 product and sum rounding.", + ), + ( + "diffusion.matrix_rope", + KernelBackend.TRITON, + "rope.matrix_rope:apply_matrix_rope", + _CUDA, + "Interleaved FP32 matrix RoPE with separate product and sum rounding.", + ), ( "diffusion.fp8_rowwise", KernelBackend.TRITON, @@ -590,6 +604,8 @@ for _op, _backend, _target, _caps, _description in _SPECS: # then symbol; a new public kernel belongs here and nowhere else. # --------------------------------------------------------------------------- _EXPORTS: dict[str, str] = { + "apply_matrix_rope": "rope.matrix_rope", + "residual_gate_fp32": "modulate.residual_gate_fp32", "can_use_fp8_rowwise": "quantization.fp8_rowwise_triton", "fp8_rowwise": "quantization.fp8_rowwise_triton", "fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit", diff --git a/python/sglang/kernels/ops/diffusion/modulate/residual_gate_fp32.py b/python/sglang/kernels/ops/diffusion/modulate/residual_gate_fp32.py new file mode 100644 index 0000000000..0b64c209ec --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/modulate/residual_gate_fp32.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Residual gating with FP32 intermediates and a single final storage cast.""" + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils.custom_op import register_custom_op + + +@triton.jit +def _residual_gate_fp32_kernel( + x, + update, + gate, + out, + count, + TOKENS: tl.constexpr, + HIDDEN: tl.constexpr, + GATE_BATCH_STRIDE: tl.constexpr, + GATE_CHANNEL_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + index = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + valid = index < count + gate_index = ( + index // (TOKENS * HIDDEN) * GATE_BATCH_STRIDE + + index % HIDDEN * GATE_CHANNEL_STRIDE + ) + residual = tl.load(x + index, valid, other=0).to(tl.float32) + value = tl.load(update + index, valid, other=0).to(tl.float32) + weight = tl.load(gate + gate_index, valid, other=0).to(tl.float32) + # eager rounds the multiply to FP32, not to the input dtype or an FMA + result = residual + weight * value + tl.store(out + index, result, valid) + + +def _fake_residual_gate_fp32( + x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + return torch.empty_like(x) + + +@register_custom_op( + op_name="diffusion_residual_gate_fp32", + mutates_args=[], + fake_impl=_fake_residual_gate_fp32, +) +def _residual_gate_fp32( + x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + output = torch.empty_like(x) + if x.numel(): + with torch.cuda.device(x.device): + _residual_gate_fp32_kernel[(triton.cdiv(x.numel(), 1024),)]( + x, + update, + gate, + output, + x.numel(), + TOKENS=x.shape[1], + HIDDEN=x.shape[2], + GATE_BATCH_STRIDE=gate.stride(0) if gate.shape[0] != 1 else 0, + GATE_CHANNEL_STRIDE=gate.stride(2), + BLOCK=1024, + enable_fp_fusion=False, + ) + return output + + +def residual_gate_fp32( + x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + """Compute ``(x.float() + gate.float() * update.float()).to(x.dtype)``. + + Unlike ``residual_gate_add``, the product is not rounded to BF16/FP16. + The CUDA path accepts contiguous [B,S,D] and a [B or 1,1,D] gate. + """ + if ( + x.is_cuda + and torch.version.hip is None + and not torch.is_grad_enabled() + and x.ndim == 3 + and x.shape == update.shape + and x.is_contiguous() + and update.is_contiguous() + and x.dtype in (torch.float16, torch.bfloat16, torch.float32) + and update.dtype == x.dtype + and gate.dtype in (x.dtype, torch.float32) + and gate.device == update.device == x.device + and gate.ndim == 3 + and gate.shape[0] in (1, x.shape[0]) + and gate.shape[1:] == (1, x.shape[2]) + ): + return _residual_gate_fp32(x, update, gate) + return (x.float() + gate.float() * update.float()).to(x.dtype) diff --git a/python/sglang/kernels/ops/diffusion/rope/matrix_rope.py b/python/sglang/kernels/ops/diffusion/rope/matrix_rope.py new file mode 100644 index 0000000000..966d19865f --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/rope/matrix_rope.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Interleaved RoPE with the eager FP32 matrix-product rounding boundaries.""" + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils.custom_op import register_custom_op + + +@triton.jit +def _matrix_rope_kernel( + x, + rope, + out, + pairs, + SEQ: tl.constexpr, + HEADS: tl.constexpr, + HALF_DIM: tl.constexpr, + BATCH_STRIDE: tl.constexpr, + SEQ_STRIDE: tl.constexpr, + PAIR_STRIDE: tl.constexpr, + ROW_STRIDE: tl.constexpr, + COL_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + idx = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + valid = idx < pairs + token = idx // (HEADS * HALF_DIM) + pos = ( + token // SEQ * BATCH_STRIDE + + token % SEQ * SEQ_STRIDE + + idx % HALF_DIM * PAIR_STRIDE + ) + # preserve the input cast before FP32 products when fusing a dtype conversion + x0 = tl.load(x + 2 * idx, valid, other=0).to(out.dtype.element_ty).to(tl.float32) + x1 = ( + tl.load(x + 2 * idx + 1, valid, other=0).to(out.dtype.element_ty).to(tl.float32) + ) + r00 = tl.load(rope + pos, valid, other=0) + r01 = tl.load(rope + pos + COL_STRIDE, valid, other=0) + r10 = tl.load(rope + pos + ROW_STRIDE, valid, other=0) + r11 = tl.load(rope + pos + ROW_STRIDE + COL_STRIDE, valid, other=0) + # do not contract the products: eager materializes them before the pair sum + y0 = x0 * r00 + x1 * r01 + y1 = x0 * r10 + x1 * r11 + tl.store(out + 2 * idx, y0, valid) + tl.store(out + 2 * idx + 1, y1, valid) + + +def _fake_apply_matrix_rope( + x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None +) -> torch.Tensor: + return torch.empty_like(x, dtype=dtype) + + +@register_custom_op( + op_name="diffusion_apply_matrix_rope", + mutates_args=[], + fake_impl=_fake_apply_matrix_rope, +) +def apply_matrix_rope( + x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None +) -> torch.Tensor: + """Apply FP32 matrices to [B,S,H,D], optionally casting x to dtype first. + + The result has the cast dtype; the input rounding precedes FP32 products. + """ + dtype = x.dtype if dtype is None else dtype + if not ( + x.is_cuda + and torch.version.hip is None + and x.ndim == 4 + and x.is_contiguous() + and x.dtype in (torch.float16, torch.bfloat16, torch.float32) + and dtype in (torch.float16, torch.bfloat16, torch.float32) + ): + raise ValueError( + "matrix RoPE requires contiguous FP16/BF16/FP32 CUDA [B,S,H,D]" + ) + batch, seq, heads, dim = x.shape + if not ( + dim % 2 == 0 + and rope.device == x.device + and rope.dtype == torch.float32 + and rope.ndim == 6 + and rope.shape[0] in (1, batch) + and rope.shape[1:] == (seq, 1, dim // 2, 2, 2) + ): + raise ValueError("matrix RoPE requires matching FP32 [B or 1,S,1,D/2,2,2]") + output = torch.empty_like(x, dtype=dtype) + pairs = x.numel() // 2 + if pairs: + with torch.cuda.device(x.device): + _matrix_rope_kernel[(triton.cdiv(pairs, 512),)]( + x, + rope, + output, + pairs, + SEQ=seq, + HEADS=heads, + HALF_DIM=dim // 2, + BATCH_STRIDE=rope.stride(0) if rope.shape[0] != 1 else 0, + SEQ_STRIDE=rope.stride(1), + PAIR_STRIDE=rope.stride(3), + ROW_STRIDE=rope.stride(4), + COL_STRIDE=rope.stride(5), + BLOCK=512, + enable_fp_fusion=False, + ) + return output diff --git a/python/sglang/multimodal_gen/configs/models/dits/kandinsky6.py b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6.py new file mode 100644 index 0000000000..c138fe698a --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 joint video/audio checkpoint architecture. + +Distilled checkpoints have n_grid velocity outputs per latent channel.""" + +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +@dataclass +class Kandinsky6ArchConfig(DiTArchConfig): + """Checkpoint fields and derived dimensions for Kandinsky6Transformer3DModel.""" + + # Chained layer renames apply to checkpoint tensors and LoRA A/B/alpha keys. + param_names_mapping: dict = field( + default_factory=lambda: { + r"^(visual_transformer_blocks\.\d+)\.video_dec_block\.(.*)$": r"\1.videoT.\2", + r"^(visual_transformer_blocks\.\d+)\.audio_dec_block\.(.*)$": r"\1.audioT.\2", + r"^(.*feed_forward)\.net\.0\.proj\.(.*)$": r"\1.mlp.fc_in.\2", + r"^(.*feed_forward)\.net\.2\.(.*)$": r"\1.mlp.fc_out.\2", + r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_1\.(.*)$": r"\1.in_layer.\2", + r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_2\.(.*)$": r"\1.out_layer.\2", + r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn_norm\.(.*)$": r"\1.self_attention_norm.\2", + r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn\.(.*)$": r"\1.self_attention.\2", + } + ) + lora_param_names_mapping: dict = field( + default_factory=lambda: {r"^transformer\.(.*)$": r"\1"} + ) + + # Diffusers Kandinsky6Transformer3DModel config fields (mirror + # transformer/config.json 1:1). + in_visual_dim: int = 16 + out_visual_dim: int = 16 + in_text_dim: int = 3584 # Qwen2.5-7B hidden size (Reason1 text encoder) + in_text_dim2: int = 768 # CLIP pooled dim + time_dim: int = 1024 + patch_size: tuple[int, int, int] = (1, 2, 2) + # checkpoint RoPE scaling is fixed across request resolutions + scale_factor: tuple[float, float, float] = (1.0, 2.0, 2.0) + model_dim: int = 4096 + ff_dim: int = 16384 + num_text_blocks: int = 4 + num_visual_blocks: int = 60 + axes_dims: tuple[int, int, int] = (32, 48, 48) # 3D RoPE T/H/W split + visual_cond: bool = True + + # False selects the video-only architecture; TI2VA pipelines use True + is_multimodal: bool = True + out_audio_dim: int | None = None + in_audio_dim: int = 20 + model_dim_a: int | None = None + time_dim_a: int | None = None + ff_dim_a: int | None = None + axes_dims_a: tuple[int, int, int] | None = None + audio_freqs_scaling: float = 1.0 + + # checkpoint metadata, not a runtime backend override; NABLA is rejected + attention_engine: str = "auto" + attention_causal: bool | None = None + attention_local: bool | None = None + attention_glob: bool | None = None + attention_window: int | None = None + attention_P: float | None = None + attention_wT: int | None = None + attention_wW: int | None = None + attention_wH: int | None = None + attention_add_sta: bool | None = None + attention_method: str | None = None + + # checkpoint metadata; the pipeline passes unpadded text rather than masks + text_token_padding: bool = False + + # Video<->audio fused block knobs (Kandinsky6FusedTransformerDecoderBlock). + ca_rope: bool = False + cross_gates: bool = False + fix_modulation: bool = False + # generated/reference token embeddings for tail-frame image conditioning + visual_token_type_num_embeddings: int = 2 + + # Derived, DiTArchConfig-contract fields (set in __post_init__ below; + # not part of the diffusers config.json). + in_channels: int = 0 + out_channels: int = 0 + + def __post_init__(self) -> None: + super().__post_init__() + if isinstance(self.patch_size, list): + self.patch_size = tuple(self.patch_size) + if len(self.patch_size) != 3: + raise ValueError(f"patch_size must have 3 values, got {self.patch_size}.") + if isinstance(self.scale_factor, list): + self.scale_factor = tuple(self.scale_factor) + if len(self.scale_factor) != 3: + raise ValueError( + f"scale_factor must have 3 values, got {self.scale_factor}." + ) + self.scale_factor = tuple(float(value) for value in self.scale_factor) + + head_dim = sum(self.axes_dims) + if self.model_dim % head_dim != 0: + raise ValueError( + f"model_dim ({self.model_dim}) must be divisible by head_dim " + f"({head_dim})" + ) + self.hidden_size = self.model_dim + self.num_attention_heads = self.model_dim // head_dim + self.in_channels = self.in_visual_dim + self.out_channels = self.out_visual_dim + self.num_channels_latents = self.in_visual_dim + + # resolve omitted audio dimensions from the video tower + self.model_dim_a = self.model_dim_a or self.model_dim + self.time_dim_a = self.time_dim_a or self.time_dim + self.ff_dim_a = self.ff_dim_a or self.ff_dim + self.axes_dims_a = self.axes_dims_a or self.axes_dims + head_dim_a = sum(self.axes_dims_a) + if self.model_dim_a % head_dim_a != 0: + raise ValueError( + f"model_dim_a ({self.model_dim_a}) must be divisible by " + f"head_dim_a ({head_dim_a})" + ) + + # without fix_modulation, cross-modal modulation uses the other tower's time + # embedding, so video and audio time widths must match + if ( + self.is_multimodal + and not self.fix_modulation + and self.time_dim_a != self.time_dim + ): + raise ValueError( + f"time_dim_a ({self.time_dim_a}) must equal time_dim " + f"({self.time_dim}) unless fix_modulation=True " + "(Kandinsky6FusedTransformerDecoderBlock's cross-modal " + "modulation is driven by the other modality's time " + "embedding by default)." + ) + + +@dataclass +class Kandinsky6VideoAudioConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=Kandinsky6ArchConfig) + prefix: str = "Kandinsky6" + + +__all__ = ["Kandinsky6ArchConfig", "Kandinsky6VideoAudioConfig"] diff --git a/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py new file mode 100644 index 0000000000..81f9f092b4 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/dits/kandinsky6_sr.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR checkpoint architecture and sampling metadata. + +Official out_visual_dim is the total DX head width; n_grid lives in the scheduler. +Unsupported architecture options and unknown fields are rejected.""" + +from dataclasses import dataclass, field +from typing import Any + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + +WIDE_INPUT_INSTRUCT_TYPES = ("channel", "hybrid", "hybrid_anchor") +SUPPORTED_INSTRUCT_TYPES = ("noise", *WIDE_INPUT_INSTRUCT_TYPES) +# Post-load behaviour overrides the port accepts (no blind ``setattr``). +ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_params") +# Bookkeeping keys the loader leaves in the component config. +_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"}) +_SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal") +_PIFLOW_FIELDS = ( + "piflow_nfe", + "piflow_num_policy_substeps", + "piflow_final_step_size_scale", + "piflow_shift", + "piflow_eps", +) +_LQ_NOISE_TYPES = ("ddpm", "linear") + + +@dataclass +class Kandinsky6SRArchConfig(DiTArchConfig): + """Static architecture metadata of ``Kandinsky6SRTransformer3DModel``.""" + + # Map the current Diffusers checkpoint names to the native SGLang port. + param_names_mapping: dict = field( + default_factory=lambda: { + r"^(.*feed_forward)\.net\.0\.proj\.(weight|bias)$": r"\1.mlp.fc_in.\2", + r"^(.*feed_forward)\.net\.2\.(weight|bias)$": r"\1.mlp.fc_out.\2", + r"^time_embeddings\.timestep_embedder\.linear_1\.(weight|bias)$": r"time_embeddings.in_layer.\1", + r"^time_embeddings\.timestep_embedder\.linear_2\.(weight|bias)$": r"time_embeddings.out_layer.\1", + } + ) + + # --- DiffusionTransformer3D constructor keys (trained architecture) --- + in_visual_dim: int = 4 + in_text_dim: int = 3584 + in_text_dim2: int = 768 + time_dim: int = 512 + out_visual_dim: int = 4 + patch_size: tuple[int, int, int] = (1, 2, 2) + model_dim: int = 2048 + ff_dim: int = 5120 + num_text_blocks: int = 2 + num_visual_blocks: int = 32 + axes_dims: tuple[int, int, int] = (16, 24, 24) + visual_cond: bool = False + instruct_type: str | None = None + attention_params: dict[str, Any] | None = None + use_motion_score: bool = False + use_lq_modulation: bool = False + lq_modulation_sublayers: dict[str, bool] | None = None + zero_lq_in_main_path: bool = False + use_adapter: bool = False + adapter_gamma: float = 0.0 + # The reference default is True; the SR release is text-free and its config says + # so explicitly. + use_text: bool = False + use_lq_noise_cond: bool = False + + # --- optional legacy / test-only DX head / pi-Flow fields --- + # An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the + # pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is + # ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim). + n_grid: int = 1 + piflow_nfe: int | None = None + piflow_num_policy_substeps: int | None = None + piflow_final_step_size_scale: float | None = None + piflow_shift: float | None = None + piflow_eps: float | None = None + + # Post-load attribute overrides (whitelist: instruct_type, visual_cond, + # attention_params); the architecture above stays as trained. The official + # config writes ``null`` when there are none. + attribute_overrides: dict[str, Any] | None = field(default_factory=dict) + + # --- SR sampling parameters of the training config --- + # As stored in ``transformer/config.json`` (``sr_params``); expanded into the + # flat ``sr_*`` fields below in ``__post_init__``. + sr_params: dict[str, Any] | None = None + sr_visual_size: list[int] = field(default_factory=lambda: [512]) + sr_scale_factor: dict[str, list[float]] | None = None + sr_scheduler_scale: float = 5.0 + sr_lq_noise_scale: float = 0.7 + sr_lq_noise_type: str = "ddpm" + sr_lq_channel_noise_scale: float = 0.0 + sr_cap_noise_timestep: bool = False + sr_fps: int = 24 + + # DiTArchConfig contract fields, derived in __post_init__. + in_channels: int = 0 + out_channels: int = 0 + + def __post_init__(self) -> None: + super().__post_init__() + if self.attribute_overrides is None: + self.attribute_overrides = {} + if self.sr_params is not None: + self._expand_sr_params(self.sr_params) + self._reject_unknown_keys() + self._reject_unsupported_flags() + self._normalize_and_validate_shapes() + self._validate_overrides() + self._validate_sampler_fields() + self.hidden_size = self.model_dim + self.num_attention_heads = self.model_dim // sum(self.axes_dims) + self.in_channels = self.in_visual_dim + self.out_channels = self.out_visual_dim + self.num_channels_latents = self.in_visual_dim + + @property + def is_piflow(self) -> bool: + return self.piflow_nfe is not None + + @property + def head_width(self) -> int: + """Channels the DiT head emits per latent position (all grids of a DX head).""" + return self.out_visual_dim * self.n_grid + + @property + def base_out_visual_dim(self) -> int: + """Channels per latent grid; legacy n_grid configs store the base output width.""" + return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim + + @property + def trained_wide_input(self) -> bool: + """Whether the input layer was trained on ``[x | cond | mask]`` channels.""" + return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES) + + def effective_override(self, name: str) -> Any: + """Post-load value of ``name``: the override if present, else as trained.""" + if name in self.attribute_overrides: + return self.attribute_overrides[name] + return getattr(self, name) + + def requested_sparse_attention(self) -> str | None: + """Sparse attention type asked for by ``attention_params``, if any.""" + params = self.effective_override("attention_params") + if not isinstance(params, dict): + return None + for value in params.values(): + attention_type = value.get("type") if isinstance(value, dict) else None + if attention_type in _SPARSE_ATTENTION_TYPES: + return attention_type + return None + + _SR_PARAM_FIELDS = { + "visual_size": "sr_visual_size", + "scale_factor": "sr_scale_factor", + "scheduler_scale": "sr_scheduler_scale", + "lq_noise_scale": "sr_lq_noise_scale", + "lq_noise_type": "sr_lq_noise_type", + "lq_channel_noise_scale": "sr_lq_channel_noise_scale", + "cap_noise_timestep": "sr_cap_noise_timestep", + "fps": "sr_fps", + } + + def _expand_sr_params(self, sr_params: dict[str, Any]) -> None: + """Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields.""" + unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS)) + if unknown: + raise ValueError( + f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted " + f"keys: {sorted(self._SR_PARAM_FIELDS)}" + ) + for key, value in sr_params.items(): + setattr(self, self._SR_PARAM_FIELDS[key], value) + + def _reject_unknown_keys(self) -> None: + unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS) + if unknown: + raise ValueError( + "Kandinsky6SR transformer config has unknown keys " + f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config." + ) + + def _reject_unsupported_flags(self) -> None: + if self.use_text: + raise NotImplementedError( + "Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not " + "supported; only the text-free SR DiT (use_text=False) is implemented." + ) + if self.use_adapter: + raise NotImplementedError( + "Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported." + ) + if self.use_lq_modulation: + raise NotImplementedError( + "Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is " + "not supported." + ) + + def _normalize_and_validate_shapes(self) -> None: + self.patch_size = tuple(self.patch_size) + self.axes_dims = tuple(self.axes_dims) + if len(self.patch_size) != 3 or self.patch_size[0] != 1: + raise ValueError( + "Kandinsky6SR patch_size must be (1, ph, pw) with a unit temporal " + f"patch, got {self.patch_size}" + ) + if len(self.axes_dims) != 3 or any(dim % 2 for dim in self.axes_dims): + raise ValueError( + f"axes_dims must be three even sizes, got {self.axes_dims}" + ) + head_dim = sum(self.axes_dims) + if self.model_dim % head_dim != 0: + raise ValueError( + f"model_dim ({self.model_dim}) must be divisible by head_dim " + f"({head_dim}) = sum(axes_dims)" + ) + if self.model_dim % 2 != 0: + raise ValueError(f"model_dim must be even, got {self.model_dim}") + if self.n_grid < 1: + raise ValueError(f"n_grid must be >= 1, got {self.n_grid}") + + def _validate_overrides(self) -> None: + unknown = sorted( + set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST) + ) + if unknown: + raise ValueError( + f"attribute_overrides keys {unknown} are not allowed; " + f"supported keys: {list(ATTRIBUTE_OVERRIDE_WHITELIST)}" + ) + instruct_type = self.effective_override("instruct_type") + if instruct_type not in (None, *SUPPORTED_INSTRUCT_TYPES): + raise ValueError( + f"instruct_type must be one of {list(SUPPORTED_INSTRUCT_TYPES)}, " + f"got {instruct_type!r}" + ) + visual_cond = self.effective_override("visual_cond") + if not isinstance(visual_cond, bool): + raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}") + + def _validate_sampler_fields(self) -> None: + present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None] + if present and len(present) != len(_PIFLOW_FIELDS): + missing = sorted(set(_PIFLOW_FIELDS) - set(present)) + raise ValueError( + f"pi-Flow config is incomplete: {missing} missing next to {present}" + ) + if self.sr_lq_noise_type not in _LQ_NOISE_TYPES: + raise ValueError( + f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, " + f"got {self.sr_lq_noise_type!r}" + ) + + +@dataclass +class Kandinsky6SRDitConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=Kandinsky6SRArchConfig) + prefix: str = "Kandinsky6SR" + + +__all__ = [ + "ATTRIBUTE_OVERRIDE_WHITELIST", + "Kandinsky6SRArchConfig", + "Kandinsky6SRDitConfig", + "SUPPORTED_INSTRUCT_TYPES", + "WIDE_INPUT_INSTRUCT_TYPES", +] diff --git a/python/sglang/multimodal_gen/configs/models/encoders/kandinsky6_reason1.py b/python/sglang/multimodal_gen/configs/models/encoders/kandinsky6_reason1.py new file mode 100644 index 0000000000..72487dc8fb --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/encoders/kandinsky6_reason1.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Reason1 text encoder, reusing the Qwen2.5-VL runtime without its vision tower. + +Defaults describe the 3584-wide language trunk; checkpoint config overrides them.""" + +from dataclasses import dataclass, field +from typing import Any + +from sglang.multimodal_gen.configs.models.encoders.base import ( + TextEncoderArchConfig, + TextEncoderConfig, +) +from sglang.multimodal_gen.configs.models.fsdp import ( + is_embed_tokens, + is_final_norm, + is_layer, +) + + +@dataclass +class Reason1ArchConfig(TextEncoderArchConfig): + """Architecture metadata (defaults match Qwen2.5-VL-7B-Instruct).""" + + architectures: list[str] = field( + default_factory=lambda: ["Qwen2_5_VLForConditionalGeneration"] + ) + + vocab_size: int = 152064 + hidden_size: int = 3584 + intermediate_size: int = 18944 + num_hidden_layers: int = 28 + num_attention_heads: int = 28 + num_key_value_heads: int = 4 + hidden_act: str = "silu" + max_position_embeddings: int = 128000 + initializer_range: float = 0.02 + rms_norm_eps: float = 1e-6 + use_cache: bool = False + tie_word_embeddings: bool = False + attention_dropout: float = 0.0 + + rope_theta: float = 1_000_000.0 + # 3D mRoPE (temporal/height/width) section split; Qwen2.5-VL-specific. + rope_scaling: dict[str, Any] | None = field( + default_factory=lambda: {"type": "mrope", "mrope_section": [16, 24, 24]} + ) + + # Kandinsky6's text-conditioning stack always consumes the last hidden + # state (see kandinsky6_qwen_postprocess_text in + # configs/pipeline_configs/kandinsky6.py), so hidden states must be kept. + output_hidden_states: bool = True + hidden_state_skip_layer: int = 0 + + # text_len caps user tokens; tokenizer length also includes the 129-token template + text_len: int = 512 + + bos_token_id: int = 151643 + pad_token_id: int = 151643 + eos_token_id: int = 151645 + vision_start_token_id: int = 151652 + vision_end_token_id: int = 151653 + vision_token_id: int = 151654 + image_token_id: int = 151655 + video_token_id: int = 151656 + + # Same fused-projection naming as the sglang qwen2_5vl/qwen_image runtime + # encoders (MergedColumnParallelLinear "gate_up_proj", fused "qkv_proj"). + stacked_params_mapping: list[tuple[str, str, str]] = field( + default_factory=lambda: [ + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), # type: ignore + (".gate_up_proj", ".up_proj", 1), # type: ignore + ] + ) + _fsdp_shard_conditions: list = field( + default_factory=lambda: [is_layer, is_embed_tokens, is_final_norm] + ) + + +@dataclass +class Reason1Config(TextEncoderConfig): + arch_config: TextEncoderArchConfig = field(default_factory=Reason1ArchConfig) + prefix: str = "reason1" + + +__all__ = ["Reason1ArchConfig", "Reason1Config"] diff --git a/python/sglang/multimodal_gen/configs/models/upsamplers/__init__.py b/python/sglang/multimodal_gen/configs/models/upsamplers/__init__.py new file mode 100644 index 0000000000..deef5616a2 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/upsamplers/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 + +from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import ( + Kandinsky6SRLatentUpscalerEntryConfig, +) + +__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"] diff --git a/python/sglang/multimodal_gen/configs/models/upsamplers/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/models/upsamplers/kandinsky6_sr.py new file mode 100644 index 0000000000..2d2bb1c0f4 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/upsamplers/kandinsky6_sr.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Configuration of an entry in the SR latent-upscaler bank. + +Only released architectures are implemented; reject unsupported settings.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import msgspec + +# key -> (required value, value a config that omits the key stands for). The omitted-key values +# are the training code's defaults, so e.g. a config without ``input_skip`` means +# ``input_skip=True`` and is rejected. +_FIXED_KEYS: dict[str, tuple[Any, Any]] = { + "upscale_factor": (4, 4), + "dims": (3, 2), + "temporal_padding": ("replicate", "zeros"), + "upsample_mode": ("pxs_v2", "pixel_shuffle"), + "upsample_padding_mode": ("zeros", "reflect"), + "modulated_norm": (True, False), + "modulated_output_proj": (True, False), + "bare_stem": (True, False), + "input_skip": (False, True), + "global_skip": (False, False), + "grn": (False, False), + "layer_scale_init": (None, None), + "depthwise": (False, False), + "bottleneck_channels": (None, None), + "motion_attention": (None, None), + # Only read by depthwise blocks; the residual convs are always 3x3x3. + "kernel_size": (3, 3), +} +# Fixed for entries built with the x2 entry (``enable_x2_entry=true``). +_X2_FIXED_KEYS: dict[str, tuple[Any, Any]] = { + "x2_tail_mode": ("private_full", "shared"), + "x2_finisher": ("pxs_residual", "none"), +} +# Accepted without effect at inference: training-time settings, and switches that only apply to +# the pixel-shuffle upsample modes (the pxs_v2 upsample has no temporal kernel and no ICNR init). +_IGNORED_KEYS = frozenset( + { + "gradient_checkpointing", + "stochastic_depth_rate", + "loss_weight_2x", + "loss_weight_4x", + "temporal_mix", + "icnr", + "upsample_position", + } +) +_REQUIRED_INT_KEYS = ( + "in_channels", + "hidden_channels", + "num_pre_blocks", + "num_mid_blocks", + "num_post_blocks", + "expand_ratio", +) +_SUPPORTED_SCALES = (2, 4) + + +def _positive_int(where: str, key: str, value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{where}: `{key}` must be a positive integer, got {value!r}") + return value + + +def _parse_target_scale(where: str, value: Any) -> int: + text = str(value).removesuffix("x") + if ( + isinstance(value, bool) + or not text.isdigit() + or int(text) not in _SUPPORTED_SCALES + ): + raise ValueError( + f"{where}: `target_scale` must be one of '2x', '4x', got {value!r}" + ) + return int(text) + + +class Kandinsky6SRLatentUpscalerEntryConfig(msgspec.Struct, frozen=True, kw_only=True): + """Cascaded 2x+2x upsampler, optionally with a private x2 entry path.""" + + target_scale: int + in_channels: int + hidden_channels: int + stage_channels: tuple[int, int, int] + num_pre_blocks: int + num_mid_blocks: int + num_post_blocks: int + expand_ratio: int + enable_x2_entry: bool = False + x2_adapter_blocks: int = 0 + + @classmethod + def from_dict( + cls, spec: Any, index: int = 0 + ) -> Kandinsky6SRLatentUpscalerEntryConfig: + """Validate one ``models[index]`` entry of ``latent_upscaler/config.json``.""" + where = f"latent upscaler models[{index}]" + if ( + not isinstance(spec, Mapping) + or set(spec) != {"target_scale", "model"} + or not isinstance(spec["model"], Mapping) + ): + raise ValueError( + f"{where} must be a mapping with exactly `target_scale` and a `model` " + f"mapping, got {spec!r}" + ) + target_scale = _parse_target_scale(where, spec["target_scale"]) + model = dict(spec["model"]) + + if model.get("architecture") != "multi_scale": + raise ValueError( + f"{where}: `architecture` must be 'multi_scale', got " + f"{model.get('architecture')!r}" + ) + enable_x2_entry = model.get("enable_x2_entry", False) + if not isinstance(enable_x2_entry, bool): + raise ValueError( + f"{where}: `enable_x2_entry` must be a boolean, got {enable_x2_entry!r}" + ) + + fixed = dict(_FIXED_KEYS) + if enable_x2_entry: + fixed.update(_X2_FIXED_KEYS) + for key, (required, omitted) in fixed.items(): + value = model.get(key, omitted) + if value != required or type(value) is not type(required): + raise ValueError( + f"{where}: unsupported `{key}`={value!r}" + f"{' (omitted)' if key not in model else ''}; only {required!r} is " + "implemented" + ) + + x2_defaults = { + "x2_adapter_blocks": 0, + "x2_adapter_sources": None, + "x2_tail_mode": "shared", + "x2_finisher": "none", + } + known = { + "architecture", + "enable_x2_entry", + "stage_channels", + *_REQUIRED_INT_KEYS, + *fixed, + *x2_defaults, + *_IGNORED_KEYS, + } + unknown = sorted(set(model) - known) + if unknown: + raise ValueError(f"{where}: unknown keys {unknown}") + if not enable_x2_entry: + present = sorted( + key + for key, default in x2_defaults.items() + if model.get(key, default) != default + ) + if present: + raise ValueError(f"{where}: {present} require `enable_x2_entry`=true") + if target_scale == 2 and not enable_x2_entry: + raise ValueError( + f"{where}: `target_scale`='2x' requires `enable_x2_entry`=true" + ) + + missing = [key for key in _REQUIRED_INT_KEYS if key not in model] + if missing: + raise ValueError(f"{where}: missing keys {missing}") + ints = { + key: _positive_int(where, key, model[key]) for key in _REQUIRED_INT_KEYS + } + hidden = ints["hidden_channels"] + + raw_stages = model.get("stage_channels") + if raw_stages is None: + stage_channels = (hidden, hidden, hidden) + else: + if not isinstance(raw_stages, (list, tuple)) or len(raw_stages) != 3: + raise ValueError( + f"{where}: `stage_channels` must list 3 widths, got {raw_stages!r}" + ) + first, second, third = ( + _positive_int(where, "stage_channels", width) for width in raw_stages + ) + stage_channels = (first, second, third) + if stage_channels[0] != hidden: + raise ValueError( + f"{where}: `hidden_channels` ({hidden}) must equal `stage_channels[0]` " + f"({stage_channels[0]})" + ) + + x2_adapter_blocks = 0 + if enable_x2_entry: + x2_adapter_blocks = _positive_int( + where, "x2_adapter_blocks", model.get("x2_adapter_blocks", 0) + ) + # Training warm-starts the adapter from these pre_blocks; only their consistency + # matters here. + sources = model.get("x2_adapter_sources") + if sources is not None and ( + not isinstance(sources, (list, tuple)) + or len(sources) != x2_adapter_blocks + or any( + not isinstance(i, int) or not 0 <= i < ints["num_pre_blocks"] + for i in sources + ) + ): + raise ValueError( + f"{where}: `x2_adapter_sources` must list {x2_adapter_blocks} indices in " + f"[0, num_pre_blocks={ints['num_pre_blocks']}), got {sources!r}" + ) + + return cls( + target_scale=target_scale, + stage_channels=stage_channels, + enable_x2_entry=enable_x2_entry, + x2_adapter_blocks=x2_adapter_blocks, + **ints, + ) + + +__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"] diff --git a/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_audio.py b/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_audio.py new file mode 100644 index 0000000000..39fa96bb23 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_audio.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint configuration for the bundled MMAudio codec and BigVGAN vocoder.""" + +from dataclasses import dataclass, field +from typing import Any + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + + +@dataclass +class Kandinsky6AudioVAEArchConfig(VAEArchConfig): + mode: str = "44k" + # The two flags below are not read by the runtime module: the official + # checkpoints carry the encoder half's weights, so ``Kandinsky6AudioVAE`` + # always builds both halves of the nested MMAudioVAE. + need_vae_decoder: bool = True + need_vae_encoder: bool = False + scaling_factor: float = 0.5302 + # Reverses a checkpoint's own audio-latent shift, matching the diffusers + # reference's postprocess_audio (getattr(audio_vae, "mean_value", 0.0)); + # no real checkpoint has been observed setting this to anything but 0.0 + # yet. + mean_value: float = 0.0 + # BigVGAN-v2 hyperparameters, passed straight through to the runtime + # BigVGANV2(...) constructor unmodified. + vocoder_config: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class Kandinsky6AudioVAEConfig(VAEConfig): + arch_config: VAEArchConfig = field(default_factory=Kandinsky6AudioVAEArchConfig) + # Generation only decodes audio; the runtime module does not read these. + load_encoder: bool = False + load_decoder: bool = True + + +__all__ = ["Kandinsky6AudioVAEArchConfig", "Kandinsky6AudioVAEConfig"] diff --git a/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_sr.py new file mode 100644 index 0000000000..c73723569f --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/vaes/kandinsky6_sr.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR causal KVAE config supporting nested and flat checkpoint layouts. + +Normalize flat fields into encoder_config/decoder_config; reject unknown fields.""" + +from dataclasses import dataclass, field +from typing import Any + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig + +_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"}) + + +@dataclass +class Kandinsky6SRVAEArchConfig(VAEArchConfig): + vae_type: str = "video-kvae" + encoder_config: dict[str, Any] = field(default_factory=dict) + decoder_config: dict[str, Any] = field(default_factory=dict) + scaling_factor: float = 1.0 + spatial_factor: int = 16 + temporal_factor: int = 4 + + # Flat top-level KVAE architecture fields (current official vae/config.json layout -- see + # module docstring). Left at their None default (and ignored) when the checkpoint instead + # uses the older nested encoder_config/decoder_config layout. + in_channels: int | None = None + out_channels: int | None = None + z_channels: int | None = None + ch: int | None = None + ch_mult: list[float] | None = None + decoder_ch: int | None = None + decoder_ch_mult: list[float] | None = None + num_res_blocks: int | None = None + padding_mode: str | None = None + temporal_compress_times: int | None = None + temporal_compress_start_level: int | None = None + norm_type: str | None = None + double_z: bool | None = None + downsample_version: int | None = None + fix_pxs: bool | None = None + # Present in the flat layout but unused by Kandinsky6SRVAE (the KVAE is resolution-agnostic); + # declared only so it doesn't trip the unknown-top-level-key check below. + resolution: int | None = None + + def __post_init__(self) -> None: + unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS) + if unknown: + raise ValueError(f"Kandinsky6SR VAE config has unknown keys {unknown}") + if self.vae_type != "video-kvae": + raise ValueError( + f"Kandinsky6SRVAE supports only 'video-kvae', got {self.vae_type!r}" + ) + self.spatial_compression_ratio = self.spatial_factor + self.temporal_compression_ratio = self.temporal_factor + + if not self.encoder_config and not self.decoder_config and self.ch is not None: + shared = dict( + num_res_blocks=self.num_res_blocks, + z_channels=self.z_channels, + temporal_compress_times=self.temporal_compress_times, + temporal_compress_start_level=self.temporal_compress_start_level, + norm_type=self.norm_type, + padding_mode=self.padding_mode, + ) + self.encoder_config = { + **shared, + "ch": self.ch, + "ch_mult": self.ch_mult, + "in_channels": self.in_channels, + "double_z": self.double_z, + "downsample_version": self.downsample_version, + "fix_pxs": self.fix_pxs, + } + self.decoder_config = { + **shared, + "ch": self.decoder_ch, + "ch_mult": self.decoder_ch_mult, + "out_ch": self.out_channels, + } + + +@dataclass +class Kandinsky6SRVAEConfig(VAEConfig): + arch_config: VAEArchConfig = field(default_factory=Kandinsky6SRVAEArchConfig) + # SR stages own tile geometry; parallel tiling distributes these existing tiles + # without enabling the generic VAE tiling or spatial-shard decode paths + use_tiling: bool = False + use_temporal_tiling: bool = False + use_parallel_tiling: bool = False + use_parallel_decode: bool = False + + def get_vae_scale_factor(self) -> int: + return self.arch_config.spatial_factor + + +__all__ = ["Kandinsky6SRVAEArchConfig", "Kandinsky6SRVAEConfig"] diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py new file mode 100644 index 0000000000..80a27e8d4c --- /dev/null +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 text-to-video/audio pipeline with optional image conditioning.""" + +from dataclasses import dataclass, field +from typing import Any, Callable + +import torch + +from sglang.multimodal_gen.configs.models import EncoderConfig +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.models.encoders import ( + BaseEncoderOutput, + CLIPTextConfig, +) +from sglang.multimodal_gen.configs.models.encoders.kandinsky6_reason1 import ( + Reason1Config, +) +from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import HunyuanVAEConfig +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ModelTaskType, + PipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + _is_kandinsky6_sr, +) +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + qwen_image_postprocess_text, +) +from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy + +# checkpoint-trained template: preserve its exact bytes, including misspellings +KANDINSKY6_PROMPT_TEMPLATE = "\n".join( + [ + "<|im_start|>system\nYou are a promt engineer. Describe the video in detail.", # codespell:ignore promt + "Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.", + "Describe the location of the video, main characters or objects and their action.", + "Describe the dynamism of the video and presented actions.", + "Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.", # codespell:ignore scren + "Describe the visual effects, postprocessing and transitions if they are presented in the video.", + "Pay attention to the order of key actions shown in the scene.<|im_end|>", + "<|im_start|>user\n{}<|im_end|>", + ] +) +# Reason1 template prefix length before user tokens +KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX = 129 + + +def kandinsky6_qwen_preprocess_text(prompt: str) -> str: + if not prompt.strip(): + prompt = "." + return KANDINSKY6_PROMPT_TEMPLATE.format(prompt) + + +def kandinsky6_qwen_postprocess_text( + outputs: BaseEncoderOutput, + text_inputs, + return_attention_mask: bool = False, +): + """Strip padding and the Reason1 template prefix, then repad embeddings and mask.""" + return qwen_image_postprocess_text( + outputs, + text_inputs, + drop_idx=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX, + return_attention_mask=return_attention_mask, + ) + + +def kandinsky6_clip_postprocess_text( + outputs: BaseEncoderOutput, _text_inputs +) -> torch.Tensor: + return outputs.pooler_output + + +@dataclass +class Kandinsky6TI2VAPipelineConfig(PipelineConfig): + """Configuration for the Kandinsky6 TI2VA pipeline.""" + + task_type: ModelTaskType = ModelTaskType.TI2V + native_only_components: tuple[str, ...] = ( + "transformer", + "text_encoder", + "text_encoder_2", + "vae", + "audio_vae", + ) + + # image conditioning is handled by Kandinsky6ImageEncodingStage, not Wan TI2V preprocessing + skip_input_image_preprocess: bool = True + + # a single joint DiT consumes video and audio latents + dit_config: Kandinsky6VideoAudioConfig = field( + default_factory=Kandinsky6VideoAudioConfig + ) + dit_precision: str = "bf16" + cfg_policy: CFGPolicy = field( + default_factory=lambda: CFGPolicy(parallel_uses_serial_arithmetic=True) + ) + + # keep tiled decode: whole-clip sharding materializes a quadratic temporal mask + vae_config: HunyuanVAEConfig = field( + default_factory=lambda: HunyuanVAEConfig(parallel_decode_mode="tiled") + ) + vae_precision: str = "bf16" + vae_tiling: bool = True + + # audio_vae includes both the mel codec and vocoder + audio_vae_config: Kandinsky6AudioVAEConfig = field( + default_factory=Kandinsky6AudioVAEConfig + ) + audio_vae_precision: str = "bf16" + + # Reason1 provides token embeddings; CLIP provides pooled text conditioning + text_encoder_configs: tuple[EncoderConfig, ...] = field( + default_factory=lambda: (Reason1Config(), CLIPTextConfig()) + ) + text_encoder_precisions: tuple[str, ...] = field( + default_factory=lambda: ("bf16", "bf16") + ) + preprocess_text_funcs: tuple[Callable[[str], str] | None, ...] = field( + default_factory=lambda: (kandinsky6_qwen_preprocess_text, None) + ) + postprocess_text_funcs: tuple[Callable[..., Any], ...] = field( + default_factory=lambda: ( + kandinsky6_qwen_postprocess_text, + kandinsky6_clip_postprocess_text, + ) + ) + # Reason1: 129 template + 512 user tokens; CLIP: fixed 77-token context + text_encoder_extra_args: list[dict] = field( + default_factory=lambda: [ + dict( + max_length=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX + 512, + padding=True, + truncation=True, + ), + dict( + max_length=77, + padding="max_length", + truncation=True, + add_special_tokens=True, + ), + ] + ) + + # Shift of both official checkpoints' schedulers (FlowMatchEulerDiscreteScheduler + # for Pro-sft, PiflowScheduler for Pro-distill: scheduler_config.json "shift"). + flow_shift: float | None = 5.0 + + # audio frames = ceil(pixel_frames / fps * sample_rate / downsample_factor) + sample_fps: float = 24.0 + audio_sample_rate: int = 44100 + audio_downsample_factor: int = 1024 + + def __post_init__(self) -> None: + if len(self.text_encoder_configs) != 2: + raise ValueError( + "Kandinsky6 pipeline requires exactly 2 text encoders " + f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}." + ) + if len(self.text_encoder_precisions) != 2: + raise ValueError( + "Kandinsky6 pipeline requires exactly 2 text encoder " + f"precisions, but got {len(self.text_encoder_precisions)}." + ) + if len(self.text_encoder_extra_args) != 2: + raise ValueError( + "Kandinsky6 pipeline requires exactly 2 text encoder extra " + f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}." + ) + + # keep the video VAE encoder available for optional image conditioning + self.vae_config.load_encoder = True + self.vae_config.load_decoder = True + + def supports_disaggregation(self) -> bool: + # joint denoising currently runs monolithically + return False + + def get_text_encoder_pooler_output(self, outputs, encoder_index): + # only CLIP (index 1) returns pooler_output; Reason1 returns causal-LM output + if encoder_index == 1: + return outputs.pooler_output + return None + + def get_pos_prompt_embeds(self, batch): + return batch.prompt_embeds[0] + + def get_neg_prompt_embeds(self, batch): + return batch.negative_prompt_embeds[0] + + def tokenize_prompt(self, prompt, tokenizer, tok_kwargs) -> dict: + # Reason1 uses a multimodal processor: positional input would bind to images, + # not text. The conditioning image is handled separately by the VAE. + return tokenizer(text=prompt, **tok_kwargs) + + +def _is_kandinsky6_t2va_family(model_id: str) -> bool: + normalized = model_id.lower().replace("-", "").replace("_", "") + return "kandinsky6" in normalized and not _is_kandinsky6_sr(model_id) + + +def _is_kandinsky6_t2va_distilled(model_id: str) -> bool: + short_name = model_id.lower().rstrip("/").split("/")[-1] + return _is_kandinsky6_t2va_family(model_id) and "distill" in short_name + + +def _is_kandinsky6_t2va(model_id: str) -> bool: + short_name = model_id.lower().rstrip("/").split("/")[-1] + return _is_kandinsky6_t2va_family(model_id) and "distill" not in short_name + + +def register(): + from sglang.multimodal_gen.configs.sample.kandinsky6 import ( + Kandinsky6TI2VADistilledSamplingParams, + Kandinsky6TI2VASamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Kandinsky6TI2VADistilledSamplingParams, + pipeline_config_cls=Kandinsky6TI2VAPipelineConfig, + hf_model_paths=["kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"], + model_detectors=[_is_kandinsky6_t2va_distilled], + ) + register_configs( + sampling_param_cls=Kandinsky6TI2VASamplingParams, + pipeline_config_cls=Kandinsky6TI2VAPipelineConfig, + hf_model_paths=[ + "kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers", + "kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers", + ], + model_detectors=[_is_kandinsky6_t2va], + ) + + +__all__ = [ + "KANDINSKY6_PROMPT_TEMPLATE", + "KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX", + "Kandinsky6TI2VAPipelineConfig", + "kandinsky6_clip_postprocess_text", + "kandinsky6_qwen_postprocess_text", + "kandinsky6_qwen_preprocess_text", +] diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6_sr.py new file mode 100644 index 0000000000..4ea8d29390 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/kandinsky6_sr.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Video-to-video SR without text/audio models; source audio is preserved. + +TI2V supplies the API task type, but model-specific stages handle video_path.""" + +import re +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + ModelTaskType, + PipelineConfig, +) + + +@dataclass +class Kandinsky6SRPipelineConfig(PipelineConfig): + """Configuration for ``Kandinsky6SRPipeline``.""" + + task_type: ModelTaskType = ModelTaskType.TI2V + skip_input_image_preprocess: bool = True + + # A missing / unexpected weight, a bad config or an unsupported flag must fail the + # load loudly: the loaders only load strictly and re-raise their errors (instead of + # falling back to a diffusers ``AutoModel``) for these components. + native_only_components: tuple[str, ...] = ("transformer", "vae", "latent_upscaler") + + dit_config: Kandinsky6SRDitConfig = field(default_factory=Kandinsky6SRDitConfig) + dit_precision: str = "bf16" + vae_config: Kandinsky6SRVAEConfig = field(default_factory=Kandinsky6SRVAEConfig) + vae_precision: str = "bf16" + vae_tiling: bool = False + vae_sp: bool = False + + # Text-free: no text encoders (all four per-encoder tuples must stay equally long). + text_encoder_configs: tuple = () + text_encoder_precisions: tuple = () + preprocess_text_funcs: tuple = () + postprocess_text_funcs: tuple = () + + should_use_guidance: bool = False + enable_autocast: bool = False + + def supports_disaggregation(self) -> bool: + # The tiled loop runs in one process over shared tensors. + return False + + +__all__ = ["Kandinsky6SRPipelineConfig"] + + +def _is_kandinsky6_sr(model_id: str) -> bool: + normalized = model_id.lower().replace("-", "").replace("_", "") + return re.search(r"kandinsky6(?:\.\d+)?(?:vsr|sr|superres)", normalized) is not None + + +def register(): + from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Kandinsky6SRSamplingParams, + pipeline_config_cls=Kandinsky6SRPipelineConfig, + hf_model_paths=[ + "kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", + "kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", + ], + model_detectors=[_is_kandinsky6_sr], + ) diff --git a/python/sglang/multimodal_gen/configs/sample/kandinsky6.py b/python/sglang/multimodal_gen/configs/sample/kandinsky6.py new file mode 100644 index 0000000000..87a4cb5260 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/sample/kandinsky6.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + +@dataclass +class Kandinsky6TI2VASamplingParams(SamplingParams): + """Sampling params for Kandinsky6 TI2VA (text[+image] -> video+audio). + + Defaults of the Diffusers ``Kandinsky6TI2VAPipeline`` (the undistilled + ``Kandinsky-6.0-Pro-sft-5s-Diffusers`` checkpoint): 512x768 @ 24fps, 121 + frames ((121 - 1) // 4 + 1 == 31 latent frames at the video VAE's + temporal_compression_ratio=4), guidance_scale=5.0, 50 inference steps. + """ + + # Video parameters (Kandinsky6 TI2VA default preset) + height: int = 512 + width: int = 768 + num_frames: int = 121 + fps: int = 24 + + # SamplingParams._adjust's multi-GPU frame-count rounding + # (`orig_latent_num_frames` up to a multiple of `num_gpus`) is a second, + # independent rounding rule on top of Kandinsky6LatentPreparationStage's + # own `num_frames % 4 == 1` rounding (the diffusers reference's own + # `latent_frames = (num_frames - 1) // 4 + 1` convention -- the single + # source of truth for this model). The two compose for num_gpus > 1 and + # land on a decoded frame count neither rule intended (e.g. the + # documented default num_frames=121 becomes 125/129/125/125 decoded + # frames at 2/3/4/8 GPUs instead of Diffusers' 121). Disabling it here + # leaves Kandinsky6LatentPreparationStage's rounding as the only one + # applied, regardless of num_gpus -- matching this model's precedent for + # opting out of this same mechanism when it owns frame-count alignment + # itself (see `enable_sequence_shard=True`, which disables it for the + # same reason a few lines up in SamplingParams._adjust). + adjust_frames: bool = False + + # Denoising stage + guidance_scale: float = 5.0 + num_inference_steps: int = 50 + + negative_prompt: str = ( + "Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst " + "quality, low quality, ugly, deformed, walking backwards" + ) + + +@dataclass +class Kandinsky6TI2VADistilledSamplingParams(Kandinsky6TI2VASamplingParams): + """Sampling params for the pi-Flow distilled Kandinsky6 TI2VA checkpoint + (``Kandinsky-6.0-Pro-distill-5s-Diffusers``). + + Distilled for 10 steps (model card; its ``scheduler_config.json`` leaves + ``nfe`` unset): its ``PiflowScheduler`` runs one DiT call per step and has + no classifier-free guidance, so any guidance_scale other than 1.0 is + rejected by the denoising stage, and the negative prompt is never encoded + either. + """ + + guidance_scale: float = 1.0 + num_inference_steps: int = 10 + + +__all__ = ["Kandinsky6TI2VADistilledSamplingParams", "Kandinsky6TI2VASamplingParams"] diff --git a/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr.py b/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr.py new file mode 100644 index 0000000000..422561ce58 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sampling parameters of Kandinsky 6 video super-resolution (video -> video).""" + +import os +import time +from dataclasses import dataclass +from typing import Any + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( + SUPPORTED_RESOLUTION_SCALES, + resolve_target_hw, +) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + +SR_RESIZE_MODES = ("fit", "exact") + + +@dataclass +class Kandinsky6SRSamplingParams(SamplingParams): + """SR request options; dimensions and fps follow video_path, prompt is ignored.""" + + # No prompt-driven generation: keep the guidance defaults inert. + negative_prompt: str = "" + guidance_scale: float = 1.0 + + # 2, 4 or 2.25 (= x1.125 pixel pre-upscale, then x2 tiles). + sr_resolution_scale: float = 2.25 + # DiT calls per tile (the repo-wide convention; counts network calls, not timestep grid + # points). Flow-matching bundles want 4; a pi-Flow bundle ignores this and runs its own + # ``nfe`` instead (the denoising stage warns if this is set to something else). + num_inference_steps: int = 4 + # Tiles denoised together (memory / speed trade-off). + sr_tiles_batch_size: int = 1 + # Minimum tile overlap of the tile grid, as a fraction of the tile size. + sr_tile_min_overlap: float = 0.20 + # Optional delivery resolution: "hd", "fullhd", "2k" or "WxH" (never upscales). + sr_target_resolution: str | None = None + # "fit" keeps the aspect ratio inside the target bucket; "exact" squeezes to it. + sr_target_resize_mode: str = "fit" + + # The SR DiT is text-free: a multipart request needs no ``prompt`` field. + prompt_optional = True + + @classmethod + def video_request_extra_fields(cls) -> frozenset[str]: + # num_inference_steps is already in the base schema; only sr_* needs model_extra + return frozenset( + { + "sr_resolution_scale", + "sr_tiles_batch_size", + "sr_tile_min_overlap", + "sr_target_resolution", + "sr_target_resize_mode", + } + ) + + @classmethod + def lower_video_request_kwargs( + cls, request: Any, kwargs: dict[str, Any] + ) -> dict[str, Any]: + """Lift the ``sr_*`` extras (declared above, kept by the multipart / JSON + request layer as pydantic ``model_extra``) into the kwargs this class is + constructed from; the base class only forwards a fixed, generic field set.""" + extra = getattr(request, "model_extra", None) or {} + for name in cls.video_request_extra_fields(): + if name in extra and extra[name] is not None: + kwargs[name] = extra[name] + return kwargs + + def _validate(self): + super()._validate() + if self.sr_resolution_scale not in SUPPORTED_RESOLUTION_SCALES: + raise ValueError( + f"sr_resolution_scale must be one of {list(SUPPORTED_RESOLUTION_SCALES)}, " + f"got {self.sr_resolution_scale!r}" + ) + # num_inference_steps >= 1 is already enforced generically by the base class; the + # denoising stage additionally rejects it there with a message naming the model, and + # warns when a pi-Flow checkpoint is asked to run something other than its own nfe. + if self.sr_tiles_batch_size < 1: + raise ValueError( + f"sr_tiles_batch_size must be >= 1, got {self.sr_tiles_batch_size}" + ) + if not 0.0 <= self.sr_tile_min_overlap < 1.0: + raise ValueError( + f"sr_tile_min_overlap must be in [0, 1), got {self.sr_tile_min_overlap}" + ) + if self.sr_target_resize_mode not in SR_RESIZE_MODES: + raise ValueError( + f"sr_target_resize_mode must be one of {list(SR_RESIZE_MODES)}, " + f"got {self.sr_target_resize_mode!r}" + ) + # "exact" mode only parses the spec: a malformed tier / WxH fails early. + resolve_target_hw(self.sr_target_resolution, (1080, 1920), "exact") + + def _validate_with_pipeline_config(self, pipeline_config): + super()._validate_with_pipeline_config(pipeline_config) + if self.source_video_path() is None: + raise ValueError( + "Kandinsky6 SR needs an input video: pass --video-path (a local " + "mp4/mkv file)." + ) + + def source_video_path(self) -> str | None: + """The single input video path (a one-element list is accepted).""" + path = self.video_path + if isinstance(path, list): + if len(path) != 1: + raise ValueError( + f"Kandinsky6 SR takes exactly one video, got {len(path)} paths" + ) + path = path[0] + return path + + def _set_output_file_name(self): + path = self.source_video_path() if self.video_path is not None else None + if self.output_file_name is None and path: + stem = os.path.splitext(os.path.basename(path))[0] + stamp = time.strftime("%Y%m%d-%H%M%S") + self.output_file_name = f"{stem}_sr{self.sr_resolution_scale:g}x_{stamp}" + super()._set_output_file_name() + + +__all__ = ["Kandinsky6SRSamplingParams"] diff --git a/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr_resolution.py b/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr_resolution.py new file mode 100644 index 0000000000..85279d2a18 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/sample/kandinsky6_sr_resolution.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR resolution validation shared by sampling parameters and output stages. + +Keep this module independent of runtime to avoid the registry import cycle.""" + +from __future__ import annotations + +import logging +from typing import Literal + +logger = logging.getLogger(__name__) + +# from-video total scales (2.25 = x1.125 pixel pre-upscale + x2 tiling). +SUPPORTED_RESOLUTION_SCALES: tuple[float, ...] = (2, 4, 2.25) + +# Delivery tiers for the target resolution: (H, W) candidates per tier -- landscape 16:9, +# square, portrait 9:16; the closest-aspect entry wins. +TARGET_RESOLUTIONS: dict[str, tuple[tuple[int, int], ...]] = { + "hd": ((720, 1280), (720, 720), (1280, 720)), + "fullhd": ((1080, 1920), (1080, 1080), (1920, 1080)), + "2k": ((1440, 2560), (1440, 1440), (2560, 1440)), +} + +TargetResizeMode = Literal["fit", "exact"] + +# Log a warning when the SR and target aspect ratios differ by more than this (relative) -- +# a mild anisotropic squeeze is fine, a big one means the wrong tier for the route. +ASPECT_MISMATCH_TOLERANCE = 0.02 + + +def fit_within( + sr_hw: tuple[int, int], bucket_hw: tuple[int, int] +) -> tuple[int, int] | None: + """Fit within the bucket without upscaling; preserve aspect and codec-even sizes.""" + scale = min(bucket_hw[0] / sr_hw[0], bucket_hw[1] / sr_hw[1]) + if scale >= 1.0: + logger.warning( + "SR result %sx%s already fits inside the %sx%s bucket -- nothing to " + "downscale, keeping it as is.", + sr_hw[1], + sr_hw[0], + bucket_hw[1], + bucket_hw[0], + ) + return None + height, width = ( + min(bucket, round(side * scale / 2) * 2) + for bucket, side in zip(bucket_hw, sr_hw, strict=True) + ) + return height, width + + +def resolve_target_hw( + spec: str | None, + sr_hw: tuple[int, int], + mode: TargetResizeMode = "fit", + validate_only: bool = False, +) -> tuple[int, int] | None: + """Resolve a tier or WxH bucket to (H, W), or None for no resize. + + fit preserves aspect; exact returns the bucket dimensions. validate_only + checks syntax without needing an actual SR result.""" + if mode not in ("fit", "exact"): + raise ValueError( + f"sr_target_resize_mode must be 'fit' or 'exact', got {mode!r}" + ) + if spec is None or spec.lower() == "none": + return None + key = spec.lower() + if key in TARGET_RESOLUTIONS: + source_ratio = sr_hw[1] / sr_hw[0] + bucket = min( + TARGET_RESOLUTIONS[key], key=lambda hw: abs(hw[1] / hw[0] - source_ratio) + ) + else: + parts = key.split("x") + if len(parts) != 2 or not all(part.strip().isdigit() for part in parts): + raise ValueError( + f"sr_target_resolution must be one of {sorted(TARGET_RESOLUTIONS)} or " + f"WxH (e.g. 1280x720), got {spec!r}" + ) + width, height = (int(part) for part in parts) + bucket = (height, width) + if mode == "exact" or validate_only: + return bucket + return fit_within(sr_hw, bucket) + + +__all__ = [ + "ASPECT_MISMATCH_TOLERANCE", + "SUPPORTED_RESOLUTION_SCALES", + "TARGET_RESOLUTIONS", + "TargetResizeMode", + "fit_within", + "resolve_target_hw", +] diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 49641bfa53..652abefe5d 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -298,6 +298,10 @@ class SamplingParams: # clamped to it, so a generic 1-step warmup stays a valid request. min_num_inference_steps: ClassVar[int] = 1 + # A prompt-free pipeline (e.g. a text-free SR / video-to-video DiT) sets this to + # ``True`` so the multipart video-generation route does not require ``prompt``. + prompt_optional: ClassVar[bool] = False + height: int | None = None width: int | None = None fps: int = 24 @@ -1442,6 +1446,42 @@ class SamplingParams: "frames for the generated output." ), ) + # Kandinsky6 video super-resolution (ignored by other models) + add_argument( + "--sr-resolution-scale", + type=float, + help="Kandinsky6 SR total upscale factor: 2, 4 or 2.25 (default: 2.25).", + ) + add_argument( + "--sr-num-steps", + type=int, + help="Kandinsky6 SR Euler grid points (>= 2, default: 5); ignored by " + "pi-Flow checkpoints.", + ) + add_argument( + "--sr-tiles-batch-size", + type=int, + help="Kandinsky6 SR: number of tiles denoised together (default: 1).", + ) + add_argument( + "--sr-tile-min-overlap", + type=float, + help="Kandinsky6 SR: minimum tile overlap fraction in [0, 1) " + "(default: 0.2).", + ) + add_argument( + "--sr-target-resolution", + type=str, + help="Kandinsky6 SR delivery resolution: hd, fullhd, 2k or WxH; the " + "result is only ever downscaled to it (default: keep the SR size).", + ) + add_argument( + "--sr-target-resize-mode", + type=str, + choices=["fit", "exact"], + help="Kandinsky6 SR: 'fit' keeps the aspect ratio inside the target " + "resolution, 'exact' resizes to it exactly (default: fit).", + ) add_argument( "--action-mode", type=str, diff --git a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py index edc19f80c3..e1e8b1ef3e 100644 --- a/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py +++ b/python/sglang/multimodal_gen/runtime/cache/cache_dit_integration.py @@ -105,15 +105,14 @@ def _patch_cache_dit_similarity(): tp_sp_group = getattr(self, "_sglang_tp_sp_group", None) target_group = tp_sp_group or sp_group or tp_group - # Averaging over a one-rank group returns the input, so skip the - # collective rather than pay for a round trip that cannot change it. + # CFG/DP peers are independent; never fall back to the global group if target_group is None or dist.get_world_size(target_group) == 1: return _original_similarity( self, t1, t2, threshold=threshold, - parallelized=parallelized, + parallelized=False, prefix=prefix, ) @@ -436,7 +435,29 @@ def _build_custom_block_adapter( ) -> Optional[BlockAdapter]: """Build a manual BlockAdapter for a model absent from cache-dit's registry, or None if the class is unknown.""" - spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(transformer.__class__.__name__) + # FSDP adds a runtime subclass; retain the native model's adapter contract. + model_name = next( + ( + cls.__name__ + for cls in type(transformer).__mro__ + if cls.__name__ == "Kandinsky6Transformer3DModel" + or cls.__name__ in _CUSTOM_BLOCK_ADAPTER_SPECS + ), + None, + ) + if model_name == "Kandinsky6Transformer3DModel": + # the second stream is evolving audio, not static text conditioning + return BlockAdapter( + transformer=transformer, + blocks=transformer.visual_transformer_blocks, + forward_pattern=( + ForwardPattern.Pattern_0 + if transformer.config.is_multimodal + else ForwardPattern.Pattern_2 + ), + has_separate_cfg=has_separate_cfg, + ) + spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(model_name) if spec is None: return None blocks = getattr(transformer, spec.blocks_attr, None) @@ -608,7 +629,9 @@ def enable_cache_on_transformer( ) parallelism_config = _build_parallelism_config(sp_group, tp_group) - if parallelism_config is not None: + if parallelism_config is not None or ( + dist.is_initialized() and dist.get_world_size() > 1 + ): _patch_cache_dit_similarity() _mark_transformer_parallelized(transformer, parallelism_config, sp_group, tp_group) diff --git a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a.py b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a.py index ee19f78e00..2928a2ab4b 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a.py +++ b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a.py @@ -128,10 +128,22 @@ class IpcA2AState: def init(self, group): import ctypes + from sglang.multimodal_gen.runtime.platforms import current_platform + from sglang.multimodal_gen.runtime.platforms.cuda import ( + device_id_to_physical_device_id, + ) + self.rank = dist.get_rank(group=group) self.group = group dev = torch.cuda.current_device() peer_dev = _peer_cuda_device(group, self.rank, dev) + # Peer access alone also admits PCIe pairs, where this NVLink transport + # can stall in its GPU-side flag wait. Query the same pair on both ranks. + physical_devices = sorted( + device_id_to_physical_device_id(d) for d in (dev, peer_dev) + ) + if not current_platform.is_full_nvlink(physical_devices): + raise _Unsupported("requires an NVLink-connected GPU pair") try: has_peer_access = torch.cuda.can_device_access_peer(dev, peer_dev) except RuntimeError as e: diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py index d28c949f79..ef008a61fb 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py @@ -398,7 +398,11 @@ class DiffGenerator: save_outputs( output_batch.output, requests[0].data_type, - requests[0].fps, + ( + output_batch.fps + if output_batch.fps is not None + else requests[0].fps + ), requests[0].save_output, lambda idx: output_requests[idx].output_file_path(), audio=output_batch.audio, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index 2aa7eb00d9..fadf72ef01 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -526,14 +526,15 @@ async def create_video( # Parse model-specific multipart metadata before creating request-owned # directories or saving uploads, so malformed JSON leaves no resources. if is_multipart: - if not prompt: + sampling_params_cls = resolve_sampling_params_cls(server_args) + if not prompt and not sampling_params_cls.prompt_optional: raise HTTPException(status_code=400, detail="prompt is required") raw_form = await request.form() extra_from_form = _multipart_video_extras( raw_form, extra_body=extra_body, extra_params=extra_params, - sampling_params_cls=resolve_sampling_params_cls(server_args), + sampling_params_cls=sampling_params_cls, ) # Resolve input upload directory (may be a temp dir when saving is disabled) @@ -613,7 +614,11 @@ async def create_video( num_frames_val = form_value("num_frames", num_frames) req = VideoGenerationsRequest( - prompt=prompt, + # ``prompt`` is a required str field on VideoGenerationsRequest; it is only + # ``None`` here for a prompt-optional pipeline (the gate above already + # rejected a missing prompt for every other one), so an empty string is the + # correct substitute, not a real (ignored) prompt value. + prompt=prompt or "", enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False, input_reference=input_path, video_path=form_value("video_path", video_input_path), @@ -710,6 +715,8 @@ async def create_video( detail=f"Failed to process image source: {str(e)}", ) payload["input_reference"] = input_path + if resolve_sampling_params_cls(server_args).prompt_optional: + payload.setdefault("prompt", "") req = VideoGenerationsRequest(**payload) except Exception as e: raise HTTPException(status_code=400, detail=f"Invalid request body: {e}") diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py index be4416d414..1f6675042a 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -1245,13 +1245,13 @@ class USPAttention(nn.Module): if get_ring_parallel_world_size() > 1: if ( meta_only_pad - and q.shape[0] == 1 and self.backend == AttentionBackendEnum.FA + and not self.causal ): return self._forward_ring_tail_pad(q, k, v, attn_mask_meta) raise NotImplementedError( "USPAttention masked path supports ring parallelism only " - "for batch-1 tail-pad metadata on the FA backend." + "for non-causal tail-pad metadata on the FA backend." ) if attn_mask is not None and attn_mask.dim() != 2: raise NotImplementedError( @@ -1505,15 +1505,15 @@ class USPAttention(nn.Module): attn_impl=self.attn_impl, real_seq_len=int(attn_mask_meta["pad_start"]), ring_ws=get_ring_parallel_world_size(), - ) + ).reshape_as(q) # Match the Ulysses tail path: masked query rows read as zeros. This # rank's chunk covers global rows [rank*chunk, (rank+1)*chunk). pad_from = ( - int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[0] + int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[1] ) - if pad_from < out.shape[0]: - out[max(pad_from, 0) :].zero_() - return _usp_output_all_to_all(out.unsqueeze(0), head_dim=2) + if pad_from < out.shape[1]: + out[:, max(pad_from, 0) :].zero_() + return _usp_output_all_to_all(out, head_dim=2) @staticmethod def _gather_sharded_sequence( diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 3dda849ef5..7e1ea59d77 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -351,7 +351,7 @@ class BaseLayerWithLoRA(nn.Module): """An in-place merge is about to write the base storage; if the snapshot is a zero-copy view into it, materialize the clone now.""" if self._base_is_view: - self.cpu_weight = self.cpu_weight.clone() + self.cpu_weight = self.cpu_weight.to("cpu", copy=True) self._base_is_view = False @torch.no_grad() @@ -495,11 +495,10 @@ class BaseLayerWithLoRA(nn.Module): if self.disable_lora: return - # Zero-copy views unmerge by subtracting B@A; do not clone the base. - if not self._base_is_view: - self._ensure_base_snapshot_owned() if self.merged: self.unmerge_lora_weights() + # Only layers actually merged need a snapshot of the mutable backing store. + self._ensure_base_snapshot_owned() lora_list = self._active_lora_list() if not lora_list: @@ -612,67 +611,14 @@ class BaseLayerWithLoRA(nn.Module): ) self.base_layer.weight = nn.Parameter(new_weight_data) del old_weight - elif self._base_is_view: - self._unmerge_by_inverse() else: - current_device = self.base_layer.weight.data.device - cpu_weight_on_device = self.cpu_weight.to(current_device, non_blocking=True) - if self.base_layer.weight.data.is_inference(): - self.base_layer.weight.data = self._as_mutable_tensor( - cpu_weight_on_device - ) - else: - self.base_layer.weight.data.copy_(cpu_weight_on_device) - if ( - cpu_weight_on_device.data_ptr() - != self.base_layer.weight.data.data_ptr() - ): - del cpu_weight_on_device + # Update the offload backing store too, not a transient GPU placeholder. + # A copy keeps inference-tensor rebinding from aliasing the snapshot. + restored = self.cpu_weight.clone() + write_dense_weight(self.base_layer, self._as_mutable_tensor(restored)) self.merged = False - @torch.no_grad() - def _unmerge_by_inverse(self) -> None: - lora_list = self._active_lora_list() - if not lora_list: - raise ValueError( - "LoRA weights not set; cannot unmerge a zero-copy view by inverse." - ) - src, current_device = self._materialized_weight_src() - data = self._as_mutable_tensor(src.to(get_local_torch_device())) - target_dtype = data.dtype - if ( - self._should_merge_in_fp32(lora_list) - and data.is_floating_point() - and data.dtype != torch.float32 - ): - data = data.to(torch.float32) - inverted = [ - ( - lora_A, - lora_B, - lora_path, - -lora_strength, - lora_rank, - lora_alpha, - output_offset, - ) - for ( - lora_A, - lora_B, - lora_path, - lora_strength, - lora_rank, - lora_alpha, - output_offset, - ) in reversed(lora_list) - ] - self._merge_lora_into_data(data, inverted) - write_dense_weight( - self.base_layer, - self._as_mutable_tensor(data.to(dtype=target_dtype)), - ) - @torch.no_grad() def commit_merged_as_base(self) -> None: """Promote the currently merged weights to the permanent base. diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 80342c3cb7..794bec9c37 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -823,12 +823,12 @@ def _ring_merge_attention( ) -> tuple[torch.Tensor, torch.Tensor]: """Online-softmax combine of one more ring step's partial attention. - `step_out` is `[T, H, D]`; `step_lse` is FlashAttention's varlen LSE - layout `[H, T]`. Both are already self-normalized over their own KV + `step_out` is `[T, H, D]` or `[B, T, H, D]`; `step_lse` is + `[H, T]` or `[B, H, T]`. Both are already self-normalized over their own KV chunk, so combining chunks is the standard two-term logsumexp merge (exact up to float rounding); done in fp32 for stability. """ - step_lse = step_lse.transpose(0, 1).unsqueeze(-1).to(torch.float32) + step_lse = step_lse.transpose(-2, -1).unsqueeze(-1).to(torch.float32) step_out = step_out.to(torch.float32) if out_acc is None: return step_out, step_lse @@ -850,7 +850,8 @@ def _ring_attention_varlen( ) -> torch.Tensor: """Ring-rotated varlen attention over one rank's local packed chunk. - `q, k, v` are this rank's full local ring chunk (`ring_chunk_len` rows, + `q, k, v` are `[T, H, D]` or `[B, T, H, D]`, with equal lengths per batch. + They contain this rank's full local ring chunk (`ring_chunk_len` rows, real rows followed by however many of this chunk's rows are padding). KV is P2P-rotated around the ring one hop per step (send this step's buffer to the next rank, receive the following step's buffer from the @@ -864,7 +865,7 @@ def _ring_attention_varlen( """ ring_pg = get_sp_group().ring_group assert ring_pg is not None, "Ring process group is not initialized." - ring_chunk_len = q.shape[0] + ring_chunk_len = q.shape[-3] _, ring_rank = get_ring_ctx() # `isend`/`irecv` (unlike collectives) address peers by global rank even @@ -913,11 +914,14 @@ def _ring_attention_varlen( max(real_seq_len - src_rank * ring_chunk_len, 0), ring_chunk_len ) if remote_used > 0: - step_out, step_lse = attn_impl.forward_ring_kv_chunk( - q, - kv_bufs[cur][0, :remote_used], - kv_bufs[cur][1, :remote_used], - ) + key = kv_bufs[cur][0, ..., :remote_used, :, :] + value = kv_bufs[cur][1, ..., :remote_used, :, :] + if q.ndim == 3: + step_out, step_lse = attn_impl.forward_ring_kv_chunk(q, key, value) + else: + step_out, step_lse, *_ = attn_impl.forward( + q, key, value, attn_metadata=None, return_softmax_lse=True + ) out_acc, lse_acc = _ring_merge_attention( out_acc, lse_acc, step_out, step_lse ) diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py index bf31f2b771..7901846cb0 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py @@ -146,8 +146,8 @@ class ComponentLoader(ABC): # the list of possible name of the component in model_index.json, e.g., scheduler component_names: list[str] = [] - # diffusers or transformers - expected_library: str = "" + # diffusers or transformers (a tuple lists several accepted spellings) + expected_library: str | tuple[str, ...] = "" _loaders_registered = False @@ -681,7 +681,12 @@ class ComponentLoader(ABC): if loader_cls is not None: expected_library = loader_cls.expected_library # Assert that the library matches what's expected for this component type - assert transformers_or_diffusers == expected_library, ( + allowed = ( + (expected_library,) + if isinstance(expected_library, str) + else tuple(expected_library) + ) + assert transformers_or_diffusers in allowed, ( f"{loader_type} must be loaded from {expected_library}, got {transformers_or_diffusers}" ) loader = loader_cls() diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/latent_upscaler_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/latent_upscaler_loader.py new file mode 100644 index 0000000000..b7594e1b25 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/latent_upscaler_loader.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Loader of the ``latent_upscaler`` component of an official Kandinsky 6 SR repo.""" + +import torch +from safetensors.torch import load_file as safetensors_load_file + +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + ComponentLoader, +) +from sglang.multimodal_gen.runtime.loader.utils import _list_safetensors_files +from sglang.multimodal_gen.runtime.models.registry import ModelRegistry +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + get_diffusers_component_config, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class LatentUpscalerLoader(ComponentLoader): + """Builds the bank from ``config.json`` and loads its weights strictly. + + Every failure (unparsable model config, missing or unexpected keys) surfaces as an + error: this component has no native diffusers fallback, so the base class must not + swallow the customized-load error and try ``AutoModel``. + """ + + component_names = ["latent_upscaler"] + # The official ``save_pretrained`` writes "kandinsky6", the Hub repo "diffusers". + expected_library = ("diffusers", "kandinsky6") + + def should_raise_customized_load_error( + self, server_args: ServerArgs, component_name: str + ) -> bool: + return True + + def load_customized( + self, component_model_path: str, server_args: ServerArgs, component_name: str + ): + config = get_diffusers_component_config(component_path=component_model_path) + class_name = config.pop("_class_name", None) + if class_name is None: + raise ValueError(f"{component_model_path}/config.json has no _class_name") + bank_cls, _ = ModelRegistry.resolve_model_cls(class_name) + bank = self._build_bank(bank_cls, config, component_model_path) + + state_dict: dict[str, torch.Tensor] = {} + for path in _list_safetensors_files(component_model_path): + state_dict.update(safetensors_load_file(path)) + if not state_dict: + raise FileNotFoundError(f"no safetensors files in {component_model_path}") + bank.load_state_dict(state_dict, strict=True) + + target_device = self.target_device( + server_args.should_start_component_on_cpu(component_name) + ) + # The reference runs the LU in bf16 (weights and non-persistent buffers). + return ( + bank.eval() + .requires_grad_(False) + .to(device=target_device, dtype=torch.bfloat16) + ) + + @staticmethod + def _build_bank(bank_cls, config: dict, component_model_path: str): + for key in ("models", "scaling_factor"): + if key not in config: + raise ValueError(f"{component_model_path}/config.json lacks {key!r}") + try: + return bank_cls( + models=config["models"], + scaling_factor=config["scaling_factor"], + scales=config.get("scales", (2, 4)), + ) + except (ValueError, KeyError, TypeError) as error: + raise ValueError( + f"invalid latent upscaler config in {component_model_path}: " + f"{type(error).__name__}: {error}" + ) from error diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index 238a806101..c04c236706 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -1435,6 +1435,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): output=result.output, audio=getattr(result, "audio", None), audio_sample_rate=getattr(result, "audio_sample_rate", None), + fps=getattr(result, "fps", None), metrics=result.metrics, usage=getattr(result, "usage", None), trajectory_timesteps=getattr(result, "trajectory_timesteps", None), @@ -1450,6 +1451,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): ) -> OutputBatch: """Merge per-output batches produced by grouped execution.""" merged = OutputBatch() + merged.fps = output_batches[0].fps parts = _ExpandedOutputParts() for output_batch in output_batches: diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py index 2bad2af19b..b3407f8477 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload_components.py @@ -53,6 +53,7 @@ VAE_COMPONENT_NAMES = frozenset( "audio_vae", "vocoder", "spatial_upsampler", + "latent_upscaler", "condition_image_encoder", "delight_vae", "diffusion_decoder", diff --git a/python/sglang/multimodal_gen/runtime/managers/scheduler.py b/python/sglang/multimodal_gen/runtime/managers/scheduler.py index f2dbb6a630..2a3c82de0e 100644 --- a/python/sglang/multimodal_gen/runtime/managers/scheduler.py +++ b/python/sglang/multimodal_gen/runtime/managers/scheduler.py @@ -1122,6 +1122,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag output_batch.audio, start, end, total_items ), audio_sample_rate=output_batch.audio_sample_rate, + fps=output_batch.fps, action_pred=self._slice_batched_value( output_batch.action_pred, start, end, total_items ), diff --git a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py new file mode 100644 index 0000000000..21a9713348 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6.py @@ -0,0 +1,1431 @@ +# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo + +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 transformer with channel-last video and [B, A, D] audio latents. + +TP shards whole heads. SP shards video tokens; text and audio stay replicated. +Audio-to-video cross-attention gathers projected video K/V and removes padding. +NABLA attention is not supported. +""" + +import math +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from sglang.kernels.ops.diffusion import apply_matrix_rope, residual_gate_fp32 +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6ArchConfig, + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in +from sglang.multimodal_gen.runtime.distributed import divide, get_tp_world_size +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_ring_ctx, + get_ulysses_ctx, +) +from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import ( + gather_seq, + shard_like, + shard_seq, + tail_attn_meta, +) +from sglang.multimodal_gen.runtime.layers.activation import get_act_fn +from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.layers.layernorm import LayerNormScaleShift +from sglang.multimodal_gen.runtime.layers.linear import ( + ColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.utils import add_prefix + +logger = init_logger(__name__) + +_ARCH_CONFIG_DEFAULTS = Kandinsky6VideoAudioConfig().arch_config + +# block containers for FSDP and compilation across video-only and joint models +_KANDINSKY6_BLOCK_CONTAINERS = ( + "text_transformer_blocks", + "video_text_transformer_blocks", + "audio_text_transformer_blocks", + "visual_transformer_blocks", +) + + +def _is_kandinsky6_transformer_block(name: str, module: object) -> bool: + return is_module_list_entry_in(name, _KANDINSKY6_BLOCK_CONTAINERS) + + +def _build_rotary_freqs(dim: int, max_period: float) -> torch.Tensor: + return torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=dim, dtype=torch.float32) + / dim + ) + + +class Kandinsky6TimeEmbeddings(nn.Module): + """Sinusoidal timestep embedding -> Linear -> SiLU -> Linear.""" + + def __init__( + self, + model_dim: int, + time_dim: int, + max_period: float = 10000.0, + prefix: str = "", + ): + super().__init__() + assert model_dim % 2 == 0 + self.model_dim = model_dim + self.max_period = max_period + # plain fp32 frequencies are rebuilt by post_load_weights after meta initialization + self.freqs = _build_rotary_freqs(self.model_dim // 2, self.max_period) + self.in_layer = ReplicatedLinear( + model_dim, time_dim, bias=True, prefix=add_prefix("in_layer", prefix) + ) + self.activation = nn.SiLU() + self.out_layer = ReplicatedLinear( + time_dim, time_dim, bias=True, prefix=add_prefix("out_layer", prefix) + ) + + def forward(self, time: torch.Tensor) -> torch.Tensor: + # compute sinusoidal features in fp32, then cast to the loaded linear dtype + args = torch.outer(time, self.freqs.to(device=time.device)) + time_embed = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + time_embed = time_embed.to(self.in_layer.weight.dtype) + time_embed, _ = self.in_layer(time_embed) + time_embed = self.activation(time_embed) + time_embed, _ = self.out_layer(time_embed) + return time_embed + + +class Kandinsky6TextEmbeddings(nn.Module): + """Linear + LayerNorm projection, reused for text tokens and audio latents.""" + + def __init__(self, in_dim: int, model_dim: int, prefix: str = ""): + super().__init__() + self.in_layer = ReplicatedLinear( + in_dim, model_dim, bias=True, prefix=add_prefix("in_layer", prefix) + ) + self.norm = nn.LayerNorm(model_dim, elementwise_affine=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.in_layer(x) + return self.norm(x).type_as(x) + + +class Kandinsky6VisualEmbeddings(nn.Module): + """Patchifies ``[B, T, H, W, C]`` video into non-overlapping ``patch_size`` patches.""" + + def __init__( + self, + visual_dim: int, + model_dim: int, + patch_size: tuple[int, int, int], + prefix: str = "", + ): + super().__init__() + self.patch_size = patch_size + self.in_layer = ReplicatedLinear( + math.prod(patch_size) * visual_dim, + model_dim, + prefix=add_prefix("in_layer", prefix), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + batch_size, duration, height, width, dim = x.shape + x = ( + x.view( + batch_size, + duration // self.patch_size[0], + self.patch_size[0], + height // self.patch_size[1], + self.patch_size[1], + width // self.patch_size[2], + self.patch_size[2], + dim, + ) + .permute(0, 1, 3, 5, 2, 4, 6, 7) + .flatten(4, 7) + ) + x, _ = self.in_layer(x) + return x + + +class Kandinsky6RoPE1D(nn.Module): + """1D rotary embedding for text and audio sequences.""" + + def __init__( + self, + dim: int, + max_pos: int = 2048, + max_period: float = 10000.0, + freqs_scaling: float = 1.0, + ): + super().__init__() + self.max_period = max_period + self.dim = dim + self.max_pos = max_pos + self.freqs_scaling = freqs_scaling + freq = _build_rotary_freqs(dim // 2, max_period) * freqs_scaling + pos = torch.arange(max_pos, dtype=freq.dtype) + self.register_buffer("args", torch.outer(pos, freq), persistent=False) + + def forward(self, pos: torch.Tensor) -> torch.Tensor: + args = self.args[pos] + cosine = torch.cos(args) + sine = torch.sin(args) + rope = torch.stack([cosine, -sine, sine, cosine], dim=-1) + rope = rope.view(*rope.shape[:-1], 2, 2) + return rope.unsqueeze(-4) + + +class Kandinsky6RoPE3D(nn.Module): + """3D rotary embedding for video spatial-temporal (T, H, W) tokens.""" + + def __init__( + self, + axes_dims: tuple[int, int, int], + max_pos: tuple[int, int, int] = (128, 128, 128), + max_period: float = 10000.0, + ): + super().__init__() + self.axes_dims = axes_dims + self.max_pos = max_pos + self.max_period = max_period + + for i, (axes_dim, ax_max_pos) in enumerate( + zip(axes_dims, max_pos, strict=True) + ): + freq = _build_rotary_freqs(axes_dim // 2, max_period) + pos = torch.arange(ax_max_pos, dtype=freq.dtype) + self.register_buffer(f"args_{i}", torch.outer(pos, freq), persistent=False) + + def forward( + self, + shape: tuple[int, int, int, int], + pos: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | list[torch.Tensor], + scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0), + ) -> torch.Tensor: + batch_size, duration, height, width = shape + args_t = self.args_0[pos[0]] / scale_factor[0] + args_h = self.args_1[pos[1]] / scale_factor[1] + args_w = self.args_2[pos[2]] / scale_factor[2] + + args = torch.cat( + [ + args_t.view(1, duration, 1, 1, -1).expand( + batch_size, -1, height, width, -1 + ), + args_h.view(1, 1, height, 1, -1).expand( + batch_size, duration, -1, width, -1 + ), + args_w.view(1, 1, 1, width, -1).expand( + batch_size, duration, height, -1, -1 + ), + ], + dim=-1, + ) + cosine = torch.cos(args) + sine = torch.sin(args) + rope = torch.stack([cosine, -sine, sine, cosine], dim=-1) + rope = rope.view(*rope.shape[:-1], 2, 2) + return rope.unsqueeze(-4) + + +class Kandinsky6Modulation(nn.Module): + """AdaLN-style modulation generator: SiLU -> Linear (zero-initialized).""" + + def __init__(self, time_dim: int, model_dim: int, num_params: int): + super().__init__() + self.activation = nn.SiLU() + self.out_layer = ReplicatedLinear(time_dim, num_params * model_dim, bias=True) + self.out_layer.weight.data.zero_() + self.out_layer.bias.data.zero_() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # cast explicitly: fp32 autocast does not convert loaded bf16 weights + x = x.to(self.out_layer.weight.dtype) + x = self.activation(x) + x, _ = self.out_layer(x) + return x + + +def _apply_rotary( + x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None +) -> torch.Tensor: + dtype = x.dtype if dtype is None else dtype + if ( + x.is_cuda + and torch.version.hip is None + and not torch.is_grad_enabled() + and x.is_contiguous() + and x.dtype in (torch.float16, torch.bfloat16, torch.float32) + and rope.dtype == torch.float32 + ): + return apply_matrix_rope(x, rope, dtype) + x_ = x.to(dtype).reshape(*x.shape[:-1], -1, 1, 2).to(torch.float32) + x_out = (rope * x_).sum(dim=-1) + return x_out.reshape(*x.shape).to(dtype) + + +class Kandinsky6QKNorm(nn.RMSNorm): + def forward(self, x: torch.Tensor) -> torch.Tensor: + if ( + x.is_cuda + and torch.version.hip is None + and x.dtype == torch.float32 + and self.normalized_shape == (128,) + and not torch.is_grad_enabled() + ): + # matching dtypes enables native fusion without changing parameter + # storage; other widths retain their original reduction order + return F.rms_norm(x, self.normalized_shape, self.weight.float(), self.eps) + return super().forward(x) + + +class Kandinsky6Attention(nn.Module): + """TP-sharded attention with head-local QK norm and optional asymmetric K/V. + + Video self-attention uses Ulysses/ring. Text/audio self-attention stays + replicated; cross-attention defaults to local K/V unless explicitly gathered.""" + + def __init__( + self, + num_channels: int, + head_dim: int, + supported_attention_backends: set[AttentionBackendEnum] | None, + prefix: str = "", + kv_dim: int | None = None, + quant_config: QuantizationConfig | None = None, + is_cross_attention: bool = False, + skip_sequence_parallel: bool | None = None, + ): + super().__init__() + assert num_channels % head_dim == 0 + self.num_heads = num_channels // head_dim + kv_dim = kv_dim or num_channels + tp_size = get_tp_world_size() + self.local_num_heads = divide(self.num_heads, tp_size) + + self.to_query = ColumnParallelLinear( + num_channels, + num_channels, + bias=True, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix("to_query", prefix), + ) + self.to_key = ColumnParallelLinear( + kv_dim, + num_channels, + bias=True, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix("to_key", prefix), + ) + self.to_value = ColumnParallelLinear( + kv_dim, + num_channels, + bias=True, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix("to_value", prefix), + ) + self.query_norm = Kandinsky6QKNorm(head_dim) + self.key_norm = Kandinsky6QKNorm(head_dim) + self.out_layer = RowParallelLinear( + num_channels, + num_channels, + bias=True, + input_is_parallel=True, + quant_config=quant_config, + prefix=add_prefix("out_layer", prefix), + ) + self.attention = USPAttention( + num_heads=self.local_num_heads, + head_size=head_dim, + causal=False, + supported_attention_backends=supported_attention_backends, + is_cross_attention=is_cross_attention, + skip_sequence_parallel=( + is_cross_attention + if skip_sequence_parallel is None + else skip_sequence_parallel + ), + prefix=add_prefix("attention", prefix), + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + rotary_emb: torch.Tensor | None = None, + rotary_emb_kv: torch.Tensor | None = None, + attn_mask_meta: dict | None = None, + context_seq_len: int | None = None, + skip_sequence_parallel: bool = False, + ) -> torch.Tensor: + query, _ = self.to_query(hidden_states) + + kv_source = ( + hidden_states if encoder_hidden_states is None else encoder_hidden_states + ) + key, _ = self.to_key(kv_source) + value, _ = self.to_value(kv_source) + + # reshape rank-local whole heads after the TP projections + shape, kv_shape = query.shape[:-1], key.shape[:-1] + query = query.reshape(*shape, self.local_num_heads, -1) + key = key.reshape(*kv_shape, self.local_num_heads, -1) + value = value.reshape(*kv_shape, self.local_num_heads, -1) + + query_dtype, key_dtype = query.dtype, key.dtype + query = self.query_norm(query.float()) + if rotary_emb is not None: + query = _apply_rotary(query, rotary_emb, query_dtype) + else: + query = query.to(query_dtype) + kv_rope = ( + rotary_emb_kv + if rotary_emb_kv is not None + else (rotary_emb if encoder_hidden_states is None else None) + ) + key = self.key_norm(key.float()) + if kv_rope is not None: + key = _apply_rotary(key, kv_rope, key_dtype) + else: + key = key.to(key_dtype) + + if context_seq_len is not None: + key = gather_seq(key, context_seq_len) + value = gather_seq(value, context_seq_len) + + try: + hidden_states = self.attention( + query, + key, + value, + attn_mask_meta=attn_mask_meta, + skip_sequence_parallel_override=skip_sequence_parallel, + ) + except AssertionError as exc: + # standalone single-rank parity uses SDPA without a pipeline forward context + if "Forward context is not set" not in str(exc): + raise + + query_shape = query.shape[:-2] + key_shape = key.shape[:-2] + query = query.reshape( + query_shape[0], -1, self.local_num_heads, query.shape[-1] + ).transpose(1, 2) + key = key.reshape( + key_shape[0], -1, self.local_num_heads, key.shape[-1] + ).transpose(1, 2) + value = value.reshape( + key_shape[0], -1, self.local_num_heads, value.shape[-1] + ).transpose(1, 2) + hidden_states = F.scaled_dot_product_attention( + query, + key, + value, + attn_mask=None, + is_causal=False, + ) + hidden_states = hidden_states.transpose(1, 2).reshape( + *query_shape, self.local_num_heads, -1 + ) + + hidden_states = hidden_states.flatten(-2, -1) + hidden_states, _ = self.out_layer(hidden_states) + return hidden_states + + +class _Kandinsky6MLP(nn.Module): + """Bias-free TP MLP with checkpoint-mapped fc_in/fc_out names. + + The shared MLP currently hardcodes bias=True, so it cannot be used here.""" + + def __init__( + self, + dim: int, + ff_dim: int, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.fc_in = ColumnParallelLinear( + dim, + ff_dim, + bias=False, + gather_output=False, + quant_config=quant_config, + prefix=add_prefix("fc_in", prefix), + ) + self.act = get_act_fn("gelu") + self.fc_out = RowParallelLinear( + ff_dim, + dim, + bias=False, + input_is_parallel=True, + quant_config=quant_config, + prefix=add_prefix("fc_out", prefix), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc_in(x) + x = self.act(x) + x, _ = self.fc_out(x) + return x + + +class Kandinsky6FeedForward(nn.Module): + def __init__( + self, + dim: int, + ff_dim: int, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.mlp = _Kandinsky6MLP( + dim, ff_dim, prefix=add_prefix("mlp", prefix), quant_config=quant_config + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.mlp(x) + + +def _norm_scale_shift( + norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor +) -> torch.Tensor: + """Compute LayerNorm and AdaLN affine in fp32, then restore the input dtype.""" + return norm(x.float(), shift=shift, scale=scale).type_as(x) + + +class Kandinsky6OutLayer(nn.Module): + """Projects visual hidden states back to packed latent patches.""" + + def __init__( + self, + model_dim: int, + time_dim: int, + visual_dim: int, + patch_size: tuple[int, int, int], + ): + super().__init__() + self.patch_size = patch_size + self.modulation = Kandinsky6Modulation(time_dim, model_dim, 2) + self.norm = nn.LayerNorm(model_dim, eps=1e-5, elementwise_affine=False) + self.out_layer = ReplicatedLinear( + model_dim, math.prod(patch_size) * visual_dim, bias=True + ) + + def forward( + self, visual_embed: torch.Tensor, time_embed: torch.Tensor + ) -> torch.Tensor: + shift, scale = torch.chunk( + self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 + ) + visual_embed = ( + self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0) + + shift.float()[:, None, None] + ).type_as(visual_embed) + + x, _ = self.out_layer(visual_embed) + + batch_size, duration, height, width, _ = x.shape + x = ( + x.view( + batch_size, + duration, + height, + width, + -1, + self.patch_size[0], + self.patch_size[1], + self.patch_size[2], + ) + .permute(0, 1, 5, 2, 6, 3, 7, 4) + .flatten(1, 2) + .flatten(2, 3) + .flatten(3, 4) + ) + return x + + +class Kandinsky6OutLayerAudio(nn.Module): + """Projects audio hidden states back to audio latent channels.""" + + def __init__(self, model_dim: int, time_dim: int, audio_dim: int): + super().__init__() + self.modulation = Kandinsky6Modulation(time_dim, model_dim, 2) + self.norm = nn.LayerNorm(model_dim, eps=1e-5, elementwise_affine=False) + self.out_layer = ReplicatedLinear(model_dim, audio_dim, bias=True) + + def forward( + self, audio_embed: torch.Tensor, time_embed: torch.Tensor + ) -> torch.Tensor: + shift, scale = torch.chunk( + self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 + ) + x = ( + self.norm(audio_embed.float()) * (scale.float() + 1.0) + shift.float() + ).type_as(audio_embed) + # the reference applies a second norm after modulation; preserve its arithmetic + x = self.norm(x.float()).type_as(audio_embed) + out, _ = self.out_layer(x) + return out + + +class Kandinsky6TransformerEncoderBlock(nn.Module): + """Text-only self-attention + feed-forward block (video/audio text towers).""" + + def __init__( + self, + model_dim: int, + time_dim: int, + ff_dim: int, + head_dim: int, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.text_modulation = Kandinsky6Modulation(time_dim, model_dim, 6) + + self.self_attention_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.self_attention = Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("self_attention", prefix), + quant_config=quant_config, + # text is replicated across SP ranks, so skip Ulysses even for self-attention + skip_sequence_parallel=True, + ) + + self.feed_forward_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.feed_forward = Kandinsky6FeedForward( + model_dim, + ff_dim, + prefix=add_prefix("feed_forward", prefix), + quant_config=quant_config, + ) + + def forward( + self, x: torch.Tensor, time_embed: torch.Tensor, rope: torch.Tensor + ) -> torch.Tensor: + self_attn_params, ff_params = torch.chunk( + self.text_modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 + ) + shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1) + out = _norm_scale_shift(self.self_attention_norm, x, shift, scale) + out = self.self_attention(out, rotary_emb=rope) + x = (x.float() + gate.float() * out.float()).type_as(x) + + ff_shift, ff_scale, ff_gate = torch.chunk(ff_params, 3, dim=-1) + out = _norm_scale_shift(self.feed_forward_norm, x, ff_shift, ff_scale) + out = self.feed_forward(out) + x = (x.float() + ff_gate.float() * out.float()).type_as(x) + + return x + + +class Kandinsky6TransformerDecoderBlock(nn.Module): + """Self-attention, text cross-attention and FFN. + + The joint decoder invokes these sublayers separately to insert cross-modal attention.""" + + def __init__( + self, + model_dim: int, + time_dim: int, + ff_dim: int, + head_dim: int, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 9) + + self.self_attention_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.self_attention = Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("self_attention", prefix), + quant_config=quant_config, + # Video self-attention over the (long) visual token sequence -- + # the one role this port's sequence parallelism actually shards. + ) + + self.cross_attention_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.cross_attention = Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("cross_attention", prefix), + quant_config=quant_config, + # video queries attend to replicated text K/V + is_cross_attention=True, + ) + + self.feed_forward_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.feed_forward = Kandinsky6FeedForward( + model_dim, + ff_dim, + prefix=add_prefix("feed_forward", prefix), + quant_config=quant_config, + ) + + def forward( + self, + visual_embed: torch.Tensor, + text_embed: torch.Tensor, + time_embed: torch.Tensor, + rope: torch.Tensor | None, + attn_mask_meta: dict | None = None, + ) -> torch.Tensor: + self_attn_params, cross_attn_params, ff_params = torch.chunk( + self.visual_modulation(time_embed).unsqueeze(dim=1), 3, dim=-1 + ) + + self_shift, self_scale, self_gate = torch.chunk(self_attn_params, 3, dim=-1) + visual_out = _norm_scale_shift( + self.self_attention_norm, visual_embed, self_shift, self_scale + ) + visual_out = self.self_attention( + visual_out, + rotary_emb=rope, + attn_mask_meta=attn_mask_meta, + ) + visual_embed = ( + visual_embed.float() + self_gate.float() * visual_out.float() + ).type_as(visual_embed) + + cross_shift, cross_scale, cross_gate = torch.chunk(cross_attn_params, 3, dim=-1) + visual_out = _norm_scale_shift( + self.cross_attention_norm, visual_embed, cross_shift, cross_scale + ) + visual_out = self.cross_attention(visual_out, encoder_hidden_states=text_embed) + visual_embed = ( + visual_embed.float() + cross_gate.float() * visual_out.float() + ).type_as(visual_embed) + + ff_shift, ff_scale, ff_gate = torch.chunk(ff_params, 3, dim=-1) + visual_out = _norm_scale_shift( + self.feed_forward_norm, visual_embed, ff_shift, ff_scale + ) + visual_out = self.feed_forward(visual_out) + visual_embed = ( + visual_embed.float() + ff_gate.float() * visual_out.float() + ).type_as(visual_embed) + + return visual_embed + + +def _apply_gate_sum( + x: torch.Tensor, out: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + return residual_gate_fp32(x, out, gate) + + +def _apply_scale_shift( + norm: nn.LayerNorm, x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor +) -> torch.Tensor: + return (norm(x.float()) * (scale.float() + 1.0) + shift.float()).type_as(x) + + +class Kandinsky6FusedTransformerDecoderBlock(nn.Module): + """Joint video/audio decoder with independently modulated bidirectional attention.""" + + def __init__( + self, + model_dim: int, + time_dim: int, + ff_dim: int, + head_dim: int, + model_dim_a: int, + time_dim_a: int, + ff_dim_a: int, + head_dim_a: int, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + ca_rope: bool = False, + cross_gates: bool = False, + fix_modulation: bool = False, + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.videoT = Kandinsky6TransformerDecoderBlock( + model_dim, + time_dim, + ff_dim, + head_dim, + supported_attention_backends, + prefix=add_prefix("videoT", prefix), + quant_config=quant_config, + ) + self.audioT = Kandinsky6TransformerDecoderBlock( + model_dim_a, + time_dim_a, + ff_dim_a, + head_dim_a, + supported_attention_backends, + prefix=add_prefix("audioT", prefix), + quant_config=quant_config, + ) + self.va_cross_attention = Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("va_cross_attention", prefix), + kv_dim=model_dim_a, + quant_config=quant_config, + is_cross_attention=True, + ) + self.av_cross_attention = Kandinsky6Attention( + model_dim_a, + head_dim_a, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("av_cross_attention", prefix), + kv_dim=model_dim, + quant_config=quant_config, + is_cross_attention=True, + ) + self.va_modulation = Kandinsky6Modulation( + time_dim, + model_dim if not cross_gates else model_dim * 2 + model_dim_a, + 1 if cross_gates else 3, + ) + self.av_modulation = Kandinsky6Modulation( + time_dim_a, + model_dim_a if not cross_gates else model_dim_a * 2 + model_dim, + 1 if cross_gates else 3, + ) + self.va_normalization = nn.LayerNorm(model_dim, elementwise_affine=False) + self.av_normalization = nn.LayerNorm(model_dim_a, elementwise_affine=False) + self.ca_rope = ca_rope + self.cross_gates = cross_gates + self.fix_modulation = fix_modulation + self.model_dim = model_dim + self.model_dim_a = model_dim_a + + def forward( + self, + vis: torch.Tensor, + aud: torch.Tensor, + text_v: torch.Tensor, + text_a: torch.Tensor, + time_embed: tuple[torch.Tensor, torch.Tensor], + vis_rope: torch.Tensor | None, + aud_rope: torch.Tensor | None, + va_gate_scale: float = 1.0, + av_gate_scale: float = 1.0, + video_seq_len: int | None = None, + video_attn_meta: dict | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + t_v, t_a = time_embed + + sa_p, ca_p, ff_p = torch.chunk( + self.videoT.visual_modulation(t_v).unsqueeze(dim=1), 3, dim=-1 + ) + shift, scale, gate = torch.chunk(sa_p, 3, dim=-1) + vis = _apply_gate_sum( + vis, + self.videoT.self_attention( + _norm_scale_shift(self.videoT.self_attention_norm, vis, shift, scale), + rotary_emb=vis_rope, + attn_mask_meta=video_attn_meta, + ), + gate, + ) + shift, scale, gate_v = torch.chunk(ca_p, 3, dim=-1) + vis_pre_ca = _norm_scale_shift( + self.videoT.cross_attention_norm, vis, shift, scale + ) + vis_out_t = self.videoT.cross_attention( + vis_pre_ca, encoder_hidden_states=text_v + ) + + sa_p, ca_p, ff_p_a = torch.chunk( + self.audioT.visual_modulation(t_a).unsqueeze(dim=1), 3, dim=-1 + ) + shift, scale, gate = torch.chunk(sa_p, 3, dim=-1) + aud = _apply_gate_sum( + aud, + self.audioT.self_attention( + _norm_scale_shift(self.audioT.self_attention_norm, aud, shift, scale), + rotary_emb=aud_rope, + skip_sequence_parallel=True, + ), + gate, + ) + shift, scale, gate_a = torch.chunk(ca_p, 3, dim=-1) + aud_pre_ca = _norm_scale_shift( + self.audioT.cross_attention_norm, aud, shift, scale + ) + aud_out_t = self.audioT.cross_attention( + aud_pre_ca, encoder_hidden_states=text_a + ) + aud = _apply_gate_sum(aud, aud_out_t, gate_a) + + t_va_mod = t_a if not self.fix_modulation else t_v + t_av_mod = t_v if not self.fix_modulation else t_a + va_params = self.va_modulation(t_va_mod).unsqueeze(dim=1) + av_params = self.av_modulation(t_av_mod).unsqueeze(dim=1) + if self.cross_gates: + va_shift, va_scale, va_gate = torch.split( + va_params, [self.model_dim, self.model_dim, self.model_dim_a], dim=-1 + ) + av_shift, av_scale, av_gate = torch.split( + av_params, [self.model_dim_a, self.model_dim_a, self.model_dim], dim=-1 + ) + else: + va_shift, va_scale, va_gate = torch.chunk(va_params, 3, dim=-1) + av_shift, av_scale, av_gate = torch.chunk(av_params, 3, dim=-1) + + vis = _apply_gate_sum(vis, vis_out_t, gate_v) + vis_for_va = _apply_scale_shift(self.va_normalization, vis, va_scale, va_shift) + aud_for_av = _apply_scale_shift(self.av_normalization, aud, av_scale, av_shift) + rq_v = vis_rope if self.ca_rope else None + rk_a = aud_rope if self.ca_rope else None + vis_from_aud = self.va_cross_attention( + vis_for_va, + encoder_hidden_states=aud_pre_ca, + rotary_emb=rq_v, + rotary_emb_kv=rk_a, + ) + aud_from_vis = self.av_cross_attention( + aud_for_av, + encoder_hidden_states=vis_pre_ca, + rotary_emb=rk_a, + rotary_emb_kv=rq_v, + context_seq_len=video_seq_len, + ) + vis = _apply_gate_sum( + vis, + vis_from_aud, + (va_gate if not self.cross_gates else av_gate) * va_gate_scale, + ) + aud = _apply_gate_sum( + aud, + aud_from_vis, + (av_gate if not self.cross_gates else va_gate) * av_gate_scale, + ) + + shift, scale, gate = torch.chunk(ff_p, 3, dim=-1) + vis = _apply_gate_sum( + vis, + self.videoT.feed_forward( + _norm_scale_shift(self.videoT.feed_forward_norm, vis, shift, scale) + ), + gate, + ) + shift, scale, gate = torch.chunk(ff_p_a, 3, dim=-1) + aud = _apply_gate_sum( + aud, + self.audioT.feed_forward( + _norm_scale_shift(self.audioT.feed_forward_norm, aud, shift, scale) + ), + gate, + ) + return vis, aud + + +@dataclass +class Kandinsky6TransformerOutput: + sample: torch.Tensor | tuple[torch.Tensor, torch.Tensor] + + +class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): + """Native sglang-diffusion implementation of the Kandinsky6 T2VA/IT2VA transformer.""" + + _fsdp_shard_conditions = [_is_kandinsky6_transformer_block] + _compile_conditions = [_is_kandinsky6_transformer_block] + param_names_mapping = _ARCH_CONFIG_DEFAULTS.param_names_mapping + reverse_param_names_mapping = _ARCH_CONFIG_DEFAULTS.reverse_param_names_mapping + lora_param_names_mapping = _ARCH_CONFIG_DEFAULTS.lora_param_names_mapping + # dense backends only: these attention roles do not supply sparsity metadata + _supported_attention_backends = { + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + } + + @staticmethod + def _validate_tp_config( + *, arch: Kandinsky6ArchConfig, tp_size: int, num_heads: int, num_heads_a: int + ) -> None: + if tp_size <= 0: + raise ValueError("Kandinsky6 TP size must be positive.") + for name, value in ( + ("num_attention_heads (video)", num_heads), + ("num_attention_heads_a (audio)", num_heads_a), + ("ff_dim", arch.ff_dim), + ("ff_dim_a", arch.ff_dim_a), + ): + if value % tp_size: + raise ValueError( + f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}." + ) + + @staticmethod + def _validate_sequence_parallel_config( + *, tp_size: int, num_heads: int, ulysses_size: int, ring_size: int + ) -> None: + if ulysses_size <= 0: + raise ValueError("Kandinsky6 Ulysses size must be positive.") + if ring_size <= 0: + raise ValueError("Kandinsky6 ring size must be positive.") + if ulysses_size == 1 and ring_size == 1: + return + # only Ulysses splits heads; ring rotates complete K/V shards + local_heads = num_heads // tp_size + if local_heads % ulysses_size: + raise ValueError( + f"Kandinsky6 TP-local video attention heads {local_heads} must be " + f"divisible by Ulysses size {ulysses_size} (total video heads=" + f"{num_heads}, TP={tp_size})." + ) + + def __init__( + self, + config: Kandinsky6VideoAudioConfig, + hf_config: dict[str, Any], + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__(config=config, hf_config=hf_config) + arch: Kandinsky6ArchConfig = self.config + self.quant_config = quant_config + + if arch.attention_engine == "nabla": + raise NotImplementedError( + "Kandinsky6Transformer3DModel: attention_engine='nabla' (NABLA block-sparse " + "attention) is not yet ported to sglang-diffusion. The only verified Kandinsky6 " + "checkpoint uses attention_engine='sdpa'; set attention_engine to 'auto' or " + "'sdpa' to use the dense attention path." + ) + + head_dim = sum(arch.axes_dims) + head_dim_a = sum(arch.axes_dims_a) + + tp_size = get_tp_world_size() + ulysses_size, _ = get_ulysses_ctx() + ring_size, _ = get_ring_ctx() + self._validate_tp_config( + arch=arch, + tp_size=tp_size, + num_heads=arch.model_dim // head_dim, + num_heads_a=arch.model_dim_a // head_dim_a, + ) + self._validate_sequence_parallel_config( + tp_size=tp_size, + num_heads=arch.model_dim // head_dim, + ulysses_size=ulysses_size, + ring_size=ring_size, + ) + self.in_visual_dim = arch.in_visual_dim + self.in_audio_dim = arch.in_audio_dim + self.model_dim = arch.model_dim + self.patch_size = arch.patch_size + self.visual_cond = arch.visual_cond + self.is_multimodal = arch.is_multimodal + self.attention_engine = arch.attention_engine + self.visual_token_type_num_embeddings = arch.visual_token_type_num_embeddings + + visual_embed_dim = ( + (2 * arch.in_visual_dim + 1) if arch.visual_cond else arch.in_visual_dim + ) + + self.visual_embeddings = Kandinsky6VisualEmbeddings( + visual_embed_dim, + arch.model_dim, + arch.patch_size, + prefix=add_prefix("visual_embeddings", self.prefix), + ) + self.visual_token_type_embeddings: nn.Embedding | None = None + if self.visual_token_type_num_embeddings > 0: + self.visual_token_type_embeddings = nn.Embedding( + self.visual_token_type_num_embeddings, arch.model_dim + ) + self.visual_rope_embeddings = Kandinsky6RoPE3D(arch.axes_dims) + self.out_layer = Kandinsky6OutLayer( + arch.model_dim, arch.time_dim, arch.out_visual_dim, arch.patch_size + ) + + if not self.is_multimodal: + self.time_embeddings = Kandinsky6TimeEmbeddings( + arch.model_dim, arch.time_dim + ) + self.text_embeddings = Kandinsky6TextEmbeddings( + arch.in_text_dim, arch.model_dim + ) + self.pooled_text_embeddings = Kandinsky6TextEmbeddings( + arch.in_text_dim2, arch.time_dim + ) + self.text_rope_embeddings = Kandinsky6RoPE1D(head_dim) + self.text_transformer_blocks = nn.ModuleList( + [ + Kandinsky6TransformerEncoderBlock( + arch.model_dim, + arch.time_dim, + arch.ff_dim, + head_dim, + self._supported_attention_backends, + prefix=add_prefix(f"text_transformer_blocks.{i}", self.prefix), + quant_config=quant_config, + ) + for i in range(arch.num_text_blocks) + ] + ) + self.visual_transformer_blocks = nn.ModuleList( + [ + Kandinsky6TransformerDecoderBlock( + arch.model_dim, + arch.time_dim, + arch.ff_dim, + head_dim, + self._supported_attention_backends, + prefix=add_prefix( + f"visual_transformer_blocks.{i}", self.prefix + ), + quant_config=quant_config, + ) + for i in range(arch.num_visual_blocks) + ] + ) + else: + self.audio_embeddings = Kandinsky6TextEmbeddings( + arch.in_audio_dim, arch.model_dim_a + ) + self.audio_rope_embeddings = Kandinsky6RoPE1D( + head_dim_a, freqs_scaling=arch.audio_freqs_scaling + ) + self.audio_out_layer = Kandinsky6OutLayerAudio( + arch.model_dim_a, + arch.time_dim_a, + arch.out_audio_dim or arch.in_audio_dim, + ) + + for ( + tower_prefix, + tower_model_dim, + tower_time_dim, + tower_head_dim, + tower_ff_dim, + ) in ( + ("video", arch.model_dim, arch.time_dim, head_dim, arch.ff_dim), + ("audio", arch.model_dim_a, arch.time_dim_a, head_dim_a, arch.ff_dim_a), + ): + setattr( + self, + f"{tower_prefix}_time_embeddings", + Kandinsky6TimeEmbeddings(tower_model_dim, tower_time_dim), + ) + setattr( + self, + f"{tower_prefix}_text_embeddings", + Kandinsky6TextEmbeddings(arch.in_text_dim, tower_model_dim), + ) + setattr( + self, + f"{tower_prefix}_pooled_text_embeddings", + Kandinsky6TextEmbeddings(arch.in_text_dim2, tower_time_dim), + ) + setattr( + self, + f"{tower_prefix}_text_rope_embeddings", + Kandinsky6RoPE1D(tower_head_dim), + ) + setattr( + self, + f"{tower_prefix}_text_transformer_blocks", + nn.ModuleList( + [ + Kandinsky6TransformerEncoderBlock( + tower_model_dim, + tower_time_dim, + tower_ff_dim, + tower_head_dim, + self._supported_attention_backends, + prefix=add_prefix( + f"{tower_prefix}_text_transformer_blocks.{i}", + self.prefix, + ), + quant_config=quant_config, + ) + for i in range(arch.num_text_blocks) + ] + ), + ) + + self.visual_transformer_blocks = nn.ModuleList( + [ + Kandinsky6FusedTransformerDecoderBlock( + arch.model_dim, + arch.time_dim, + arch.ff_dim, + head_dim, + arch.model_dim_a, + arch.time_dim_a, + arch.ff_dim_a, + head_dim_a, + self._supported_attention_backends, + prefix=add_prefix( + f"visual_transformer_blocks.{i}", self.prefix + ), + ca_rope=arch.ca_rope, + cross_gates=arch.cross_gates, + fix_modulation=arch.fix_modulation, + quant_config=quant_config, + ) + for i in range(arch.num_visual_blocks) + ] + ) + + self.gradient_checkpointing = False + self.hidden_size = arch.hidden_size + self.num_attention_heads = arch.num_attention_heads + self.num_channels_latents = arch.num_channels_latents + self.layer_names = ( + ["video_text_transformer_blocks", "audio_text_transformer_blocks"] + if self.is_multimodal + else ["text_transformer_blocks"] + ) + ["visual_transformer_blocks"] + self.__post_init__() + + def _encode_text( + self, + prefix: str | None, + text_embed: torch.Tensor, + pooled: torch.Tensor, + time: torch.Tensor, + text_rope_pos: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + prefix = f"{prefix}_" if prefix else "" + te = self.get_submodule(f"{prefix}text_embeddings")(text_embed) + tm = self.get_submodule(f"{prefix}time_embeddings")(time) + tm = tm + self.get_submodule(f"{prefix}pooled_text_embeddings")(pooled) + rope_embeddings = self.get_submodule(f"{prefix}text_rope_embeddings") + blocks = self.get_submodule(f"{prefix}text_transformer_blocks") + # each text tower has its own head dimension and RoPE table + text_rope = rope_embeddings(text_rope_pos).unsqueeze(dim=0) + for block in blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + te = torch.utils.checkpoint.checkpoint( + block, te, tm, text_rope, use_reentrant=False + ) + else: + te = block(te, tm, text_rope) + return te, tm + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + timestep: torch.Tensor, + encoder_hidden_states_image: torch.Tensor | None = None, + pooled_projections: torch.Tensor | None = None, + hidden_states_audio: torch.Tensor | None = None, + audio_timestep: torch.Tensor | None = None, + visual_rope_pos: ( + tuple[torch.Tensor, torch.Tensor, torch.Tensor] | list[torch.Tensor] | None + ) = None, + audio_rope_pos: torch.Tensor | None = None, + text_rope_pos: torch.Tensor | None = None, + scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0), + sparse_params: dict[str, Any] | None = None, + visual_token_type_ids: torch.Tensor | None = None, + va_gate_scale: float = 1.0, + av_gate_scale: float = 1.0, + return_dict: bool = False, + **kwargs: Any, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | Kandinsky6TransformerOutput: + if pooled_projections is None: + if encoder_hidden_states_image is None: + raise ValueError("pooled_projections must be provided for Kandinsky6.") + pooled_projections = encoder_hidden_states_image + if visual_rope_pos is None or text_rope_pos is None: + raise ValueError( + "visual_rope_pos and text_rope_pos are required for Kandinsky6." + ) + if ( + visual_token_type_ids is not None + and self.visual_token_type_num_embeddings == 0 + ): + raise ValueError( + "visual_token_type_ids requires visual_token_type_num_embeddings > 0." + ) + + if sparse_params is not None: + raise NotImplementedError( + "Kandinsky6 does not support NABLA sparse attention." + ) + both = self.is_multimodal + if both and (hidden_states is None or hidden_states_audio is None): + raise NotImplementedError( + "Multimodal Kandinsky6 checkpoints require both video and audio latents." + ) + + video_te, video_tm = self._encode_text( + "video" if both else None, + encoder_hidden_states, + pooled_projections, + timestep, + text_rope_pos, + ) + if both: + audio_te, audio_tm = self._encode_text( + "audio", + encoder_hidden_states, + pooled_projections, + audio_timestep if audio_timestep is not None else timestep, + text_rope_pos, + ) + if audio_rope_pos is None: + audio_rope_pos = torch.arange( + hidden_states_audio.shape[1], device=hidden_states_audio.device + ) + + visual_embed = self.visual_embeddings(hidden_states) + if ( + self.visual_token_type_embeddings is not None + and visual_token_type_ids is not None + ): + type_embed = self.visual_token_type_embeddings(visual_token_type_ids) + visual_embed = visual_embed + type_embed[:, :, None, None, :] + visual_shape = visual_embed.shape[:-1] + visual_rope = self.visual_rope_embeddings( + visual_shape, visual_rope_pos, scale_factor + ).flatten(1, 3) + visual_embed, shard = shard_seq(visual_embed.flatten(1, 3)) + visual_rope = shard_like(visual_rope, shard, pad_mode="repeat_last") + attn_meta = tail_attn_meta(shard, visual_embed.shape[0], visual_embed.device) + + if both: + audio_embed = self.audio_embeddings(hidden_states_audio) + audio_rope = self.audio_rope_embeddings(audio_rope_pos).unsqueeze(dim=0) + for block in self.visual_transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + visual_embed, audio_embed = torch.utils.checkpoint.checkpoint( + block, + visual_embed, + audio_embed, + video_te, + audio_te, + (video_tm, audio_tm), + visual_rope, + audio_rope, + va_gate_scale, + av_gate_scale, + video_seq_len=shard.orig_len, + video_attn_meta=attn_meta, + use_reentrant=False, + ) + else: + visual_embed, audio_embed = block( + visual_embed, + audio_embed, + video_te, + audio_te, + (video_tm, audio_tm), + visual_rope, + audio_rope, + va_gate_scale, + av_gate_scale, + video_seq_len=shard.orig_len, + video_attn_meta=attn_meta, + ) + else: + for block in self.visual_transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + visual_embed = torch.utils.checkpoint.checkpoint( + block, + visual_embed, + video_te, + video_tm, + visual_rope, + attn_mask_meta=attn_meta, + use_reentrant=False, + ) + else: + visual_embed = block( + visual_embed, video_te, video_tm, visual_rope, attn_meta + ) + + visual_embed = gather_seq(visual_embed, shard.orig_len).reshape( + *visual_shape, -1 + ) + video_out = self.out_layer(visual_embed, video_tm) + result = ( + (video_out, self.audio_out_layer(audio_embed, audio_tm)) + if both + else video_out + ) + + if return_dict: + return Kandinsky6TransformerOutput(sample=result) + return result + + def post_load_weights(self) -> None: + """Materialize non-checkpoint RoPE/time frequencies left on meta after loading.""" + device = next(self.parameters()).device + + for i, (axes_dim, ax_max_pos) in enumerate( + zip( + self.visual_rope_embeddings.axes_dims, + self.visual_rope_embeddings.max_pos, + strict=True, + ) + ): + name = f"args_{i}" + buf = self.visual_rope_embeddings._buffers.get(name) + if isinstance(buf, torch.Tensor) and buf.is_meta: + freq = _build_rotary_freqs( + axes_dim // 2, self.visual_rope_embeddings.max_period + ).to(device=device) + pos = torch.arange(ax_max_pos, dtype=freq.dtype, device=device) + self.visual_rope_embeddings._buffers[name] = torch.outer(pos, freq) + + for module in self.modules(): + if isinstance(module, Kandinsky6RoPE1D) and module.args.is_meta: + freq = ( + _build_rotary_freqs(module.dim // 2, module.max_period).to( + device=device + ) + * module.freqs_scaling + ) + pos = torch.arange(module.max_pos, dtype=freq.dtype, device=device) + module._buffers["args"] = torch.outer(pos, freq) + elif isinstance(module, Kandinsky6TimeEmbeddings) and module.freqs.is_meta: + module.freqs = _build_rotary_freqs( + module.model_dim // 2, module.max_period + ).to(device=device) + + +EntryClass = Kandinsky6Transformer3DModel diff --git a/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py new file mode 100644 index 0000000000..d13cd7fce5 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/kandinsky6_sr.py @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Text-free Kandinsky 6 SR DiT, reusing the joint model's TP/SP building blocks. + +Latents are [B, T, H, W, C]. Residuals, norms and modulation remain fp32; +linear inputs use the parameter dtype. A pi-Flow DX head emits n_grid velocities +per latent channel. VAE, upscaler and tile scheduling remain replicated.""" + +from collections.abc import Iterable, Iterator +from typing import Any + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + ATTRIBUTE_OVERRIDE_WHITELIST, + Kandinsky6SRArchConfig, + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in +from sglang.multimodal_gen.runtime.distributed import get_tp_world_size +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_ring_ctx, + get_ulysses_ctx, +) +from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import ( + gather_seq, + shard_like, + shard_seq, + tail_attn_meta, +) +from sglang.multimodal_gen.runtime.layers.layernorm import LayerNormScaleShift +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, +) +from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( + Kandinsky6Attention, + Kandinsky6FeedForward, + Kandinsky6Modulation, + Kandinsky6OutLayer, + Kandinsky6RoPE3D, + Kandinsky6TimeEmbeddings, + Kandinsky6VisualEmbeddings, + _build_rotary_freqs, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.utils import add_prefix + +logger = init_logger(__name__) + +_ARCH_CONFIG_DEFAULTS = Kandinsky6SRDitConfig().arch_config +_BLOCK_CONTAINERS = ("visual_transformer_blocks",) +_TIME_EMBEDDING_NAMES = ("time_embeddings", "motion_embeddings", "lq_noise_embeddings") + + +def _is_sr_transformer_block(name: str, module: object) -> bool: + return is_module_list_entry_in(name, _BLOCK_CONTAINERS) + + +def _modulate( + norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor +) -> torch.Tensor: + """fp32 ``LN(x) * (scale + 1) + shift`` (no cast back: the caller picks the dtype).""" + return norm(x.float(), shift=shift, scale=scale) + + +class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D): + """Keep fp32 RoPE angle tables across residency-manager dtype casts.""" + + def _apply(self, fn, recurse=True): + device = fn(torch.zeros(1)).device + for name, buffer in self._buffers.items(): + if buffer is not None: + self._buffers[name] = buffer.to(device=device) + return self + + +class Kandinsky6SRDecoderBlock(nn.Module): + """Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears.""" + + def __init__( + self, + model_dim: int, + time_dim: int, + ff_dim: int, + head_dim: int, + supported_attention_backends: set[AttentionBackendEnum] | None = None, + prefix: str = "", + quant_config: QuantizationConfig | None = None, + ): + super().__init__() + self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6) + self.self_attention_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.self_attention = Kandinsky6Attention( + model_dim, + head_dim, + supported_attention_backends=supported_attention_backends, + prefix=add_prefix("self_attention", prefix), + quant_config=quant_config, + ) + self.feed_forward_norm = LayerNormScaleShift( + model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32 + ) + self.feed_forward = Kandinsky6FeedForward( + model_dim, + ff_dim, + prefix=add_prefix("feed_forward", prefix), + quant_config=quant_config, + ) + + def forward( + self, + visual_embed: torch.Tensor, + time_embed: torch.Tensor, + rope: torch.Tensor, + compute_dtype: torch.dtype, + attn_mask_meta: dict | None = None, + ) -> torch.Tensor: + modulation = self.visual_modulation(time_embed).unsqueeze(dim=1) + self_attn_params, ff_params = torch.chunk(modulation, 2, dim=-1) + + shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1) + out = _modulate(self.self_attention_norm, visual_embed, shift, scale) + out = self.self_attention( + out.to(compute_dtype), rotary_emb=rope, attn_mask_meta=attn_mask_meta + ) + visual_embed = visual_embed.float() + gate.float() * out.float() + + shift, scale, gate = torch.chunk(ff_params, 3, dim=-1) + out = _modulate(self.feed_forward_norm, visual_embed, shift, scale) + out = self.feed_forward(out.to(compute_dtype)) + return visual_embed + gate.float() * out.float() + + +class Kandinsky6SROutLayer(Kandinsky6OutLayer): + """K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype).""" + + def forward( + self, visual_embed: torch.Tensor, time_embed: torch.Tensor + ) -> torch.Tensor: + shift, scale = torch.chunk( + self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1 + ) + x = ( + self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0) + + shift.float()[:, None, None] + ) + x, _ = self.out_layer(x.to(self.out_layer.weight.dtype)) + + batch_size, duration, height, width, _ = x.shape + pt, ph, pw = self.patch_size + return ( + x.view(batch_size, duration, height, width, -1, pt, ph, pw) + .permute(0, 1, 5, 2, 6, 3, 7, 4) + .flatten(1, 2) + .flatten(2, 3) + .flatten(3, 4) + ) + + +class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin): + """Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head).""" + + _fsdp_shard_conditions = [_is_sr_transformer_block] + _compile_conditions = [_is_sr_transformer_block] + param_names_mapping = _ARCH_CONFIG_DEFAULTS.param_names_mapping + reverse_param_names_mapping: dict = {} + lora_param_names_mapping: dict = {} + # Dense attention only: the sparse (NABLA) path of the reference is not ported. + _supported_attention_backends = { + AttentionBackendEnum.FA, + AttentionBackendEnum.TORCH_SDPA, + } + + @staticmethod + def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None: + if tp_size <= 0: + raise ValueError("Kandinsky6SR TP size must be positive.") + for name, value in ( + ("num_attention_heads", arch.num_attention_heads), + ("ff_dim", arch.ff_dim), + ): + if value % tp_size: + raise ValueError( + f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}." + ) + + @staticmethod + def _validate_sequence_parallel_config( + *, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int + ) -> None: + if ulysses_size <= 0: + raise ValueError("Kandinsky6SR Ulysses size must be positive.") + if ring_size <= 0: + raise ValueError("Kandinsky6SR ring size must be positive.") + if ulysses_size == 1 and ring_size == 1: + return + # Ring Attention rotates whole K/V shards between ranks rather than + # splitting heads, so only Ulysses constrains head divisibility. + local_heads = arch.num_attention_heads // tp_size + if local_heads % ulysses_size: + raise ValueError( + f"Kandinsky6SR TP-local attention heads {local_heads} must be " + f"divisible by Ulysses size {ulysses_size} (total heads=" + f"{arch.num_attention_heads}, TP={tp_size})." + ) + + def __init__( + self, + config: Kandinsky6SRDitConfig, + hf_config: dict[str, Any], + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__(config=config, hf_config=hf_config) + arch: Kandinsky6SRArchConfig = self.config + self.quant_config = quant_config + head_dim = sum(arch.axes_dims) + + tp_size = get_tp_world_size() + ulysses_size, _ = get_ulysses_ctx() + ring_size, _ = get_ring_ctx() + self._validate_tp_config(arch=arch, tp_size=tp_size) + self._validate_sequence_parallel_config( + arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size + ) + + self.in_visual_dim = arch.in_visual_dim + self.base_out_visual_dim = arch.base_out_visual_dim + self.n_grid = arch.n_grid + self.model_dim = arch.model_dim + self.patch_size = arch.patch_size + self.use_motion_score = arch.use_motion_score + self.use_lq_noise_cond = arch.use_lq_noise_cond + # The input width is fixed by the *trained* configuration ... + self.visual_embed_dim = ( + 2 * arch.in_visual_dim + 1 + if arch.trained_wide_input + else arch.in_visual_dim + ) + self._build_layers(arch, head_dim=head_dim, quant_config=quant_config) + + self.hidden_size = arch.hidden_size + self.num_attention_heads = arch.num_attention_heads + self.num_channels_latents = arch.num_channels_latents + self.layer_names = list(_BLOCK_CONTAINERS) + # ... while inference behaviour comes from the post-load overrides. + self._apply_attribute_overrides(arch) + self._require_all_checkpoint_keys() + self.__post_init__() + + def _build_layers( + self, + arch: Kandinsky6SRArchConfig, + *, + head_dim: int, + quant_config: QuantizationConfig | None, + ) -> None: + self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim) + if arch.use_motion_score: + self.motion_embeddings = Kandinsky6TimeEmbeddings( + arch.model_dim, arch.time_dim + ) + if arch.use_lq_noise_cond: + # Present in the checkpoint; never used at inference (kept for loading). + self.lq_noise_embeddings = Kandinsky6TimeEmbeddings( + arch.model_dim, arch.time_dim + ) + # Constant that replaces the pooled empty-caption embedding of the text DiT. + self.pooled_bias = nn.Parameter(torch.zeros(arch.time_dim)) + self.visual_embeddings = Kandinsky6VisualEmbeddings( + self.visual_embed_dim, + arch.model_dim, + arch.patch_size, + prefix=add_prefix("visual_embeddings", self.prefix), + ) + self.visual_rope_embeddings = Kandinsky6SRRoPE3D(arch.axes_dims) + self.visual_transformer_blocks = nn.ModuleList( + [ + Kandinsky6SRDecoderBlock( + arch.model_dim, + arch.time_dim, + arch.ff_dim, + head_dim, + self._supported_attention_backends, + prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix), + quant_config=quant_config, + ) + for i in range(arch.num_visual_blocks) + ] + ) + self.out_layer = Kandinsky6SROutLayer( + arch.model_dim, + arch.time_dim, + arch.head_width, + arch.patch_size, + ) + + def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None: + self.instruct_type = arch.instruct_type + self.visual_cond = arch.visual_cond + self.attention_params = arch.attention_params + for name in ATTRIBUTE_OVERRIDE_WHITELIST: + if name in arch.attribute_overrides: + setattr(self, name, arch.attribute_overrides[name]) + self._check_input_width_matches_overrides() + sparse_type = arch.requested_sparse_attention() + if sparse_type is not None: + logger.warning( + "Kandinsky6SR: attention_params request %r sparse attention; this port " + "runs dense attention, so outputs differ from the sparse reference.", + sparse_type, + ) + + def _check_input_width_matches_overrides(self) -> None: + run_wide = self.visual_cond or self.instruct_type in ( + "channel", + "hybrid", + "hybrid_anchor", + ) + run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim + if run_width != self.visual_embed_dim: + raise ValueError( + f"Kandinsky6SR: the trained input layer takes {self.visual_embed_dim} " + f"channels but instruct_type={self.instruct_type!r} with " + f"visual_cond={self.visual_cond} feeds {run_width}; set " + "attribute_overrides.visual_cond=true to run a wide-input checkpoint " + "under instruct_type='noise'." + ) + + def _require_all_checkpoint_keys(self) -> None: + """A missing tensor must fail loudly instead of being zero-filled by the loader.""" + for param in self.parameters(): + param.missing_param_init = "error" + + def preprocess_loaded_state_dict( + self, weight_iterator: Iterable[tuple[str, torch.Tensor]] + ) -> Iterator[tuple[str, torch.Tensor]]: + """Pass tensors through, then raise if the checkpoint held unexpected keys.""" + map_name = get_param_names_mapping(self.param_names_mapping) + expected = set(self.state_dict()) + unexpected: list[str] = [] + for name, tensor in weight_iterator: + if map_name(name)[0] not in expected: + unexpected.append(name) + yield name, tensor + if unexpected: + shown = ", ".join(unexpected[:10]) + raise ValueError( + f"Kandinsky6SR checkpoint has {len(unexpected)} unexpected keys " + f"(first: {shown}); the transformer config does not match the weights." + ) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + visual_rope_pos: ( + tuple[torch.Tensor, torch.Tensor, torch.Tensor] | list[torch.Tensor] + ), + scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0), + motion_score: torch.Tensor | None = None, + ) -> torch.Tensor: + """Predict [B, T, H, W, head_width] velocities from channel-last latents. + + timesteps are scaled by 1000. DX channels are ordered grid-major; + the scheduler reshapes them into n_grid velocity predictions.""" + self._check_latent_shape(hidden_states) + compute_dtype = self.visual_embeddings.in_layer.weight.dtype + time_embed = self.time_embeddings(timestep) + if motion_score is not None and self.use_motion_score: + time_embed = time_embed + self.motion_embeddings(motion_score) + time_embed = time_embed + self.pooled_bias + + visual_embed = self.visual_embeddings(hidden_states.to(compute_dtype)) + visual_shape = visual_embed.shape[:-1] + visual_rope = self.visual_rope_embeddings( + visual_shape, visual_rope_pos, scale_factor + ) + visual_embed = visual_embed.flatten(1, 3) + visual_rope = visual_rope.flatten(1, 3) + visual_embed, shard = shard_seq(visual_embed) + visual_rope = shard_like(visual_rope, shard, pad_mode="repeat_last") + attn_meta = tail_attn_meta(shard, visual_embed.shape[0], visual_embed.device) + for block in self.visual_transformer_blocks: + visual_embed = block( + visual_embed, + time_embed, + visual_rope, + compute_dtype, + attn_mask_meta=attn_meta, + ) + visual_embed = gather_seq(visual_embed, shard.orig_len) + return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed) + + def _check_latent_shape(self, hidden_states: torch.Tensor) -> None: + if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim: + raise ValueError( + "Kandinsky6SR expects a [B, T, H, W, " + f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}" + ) + height, width = hidden_states.shape[2:4] + if height % self.patch_size[1] or width % self.patch_size[2]: + raise ValueError( + f"latent {height}x{width} is not divisible by patch {self.patch_size}" + ) + + def post_load_weights(self) -> None: + """Rebuild RoPE / time-embedding tables that a meta-device init left on meta.""" + device = next(self.parameters()).device + rope = self.visual_rope_embeddings + if any(buf.is_meta for buf in rope.buffers()): + self.visual_rope_embeddings = Kandinsky6SRRoPE3D( + rope.axes_dims, rope.max_pos, rope.max_period + ).to(device) + for name in _TIME_EMBEDDING_NAMES: + embeddings = self._modules.get(name) + if embeddings is not None and embeddings.freqs.is_meta: + embeddings.freqs = _build_rotary_freqs( + embeddings.model_dim // 2, embeddings.max_period + ).to(device=device) + + +EntryClass = Kandinsky6SRTransformer3DModel diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/clip.py b/python/sglang/multimodal_gen/runtime/models/encoders/clip.py index ce42af1d88..1c503edc3c 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/clip.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/clip.py @@ -183,6 +183,9 @@ class CLIPTextModel(TextEncoder): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() for name, loaded_weight in weights: + # standalone CLIP text-transformer checkpoints omit the model wrapper + if name.startswith(("embeddings.", "encoder.", "final_layer_norm.")): + name = f"text_model.{name}" name = _srt_clip_param_name(name) # Handle q_proj, k_proj, v_proj -> qkv_proj mapping for param_name, weight_name, shard_id in stacked_params_mapping: diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py b/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py new file mode 100644 index 0000000000..819b101770 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/kandinsky6_piflow.py @@ -0,0 +1,314 @@ +# Copyright 2025 The Kandinsky Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Diffusers scheduler for distilled Kandinsky 6 PiFlow checkpoints.""" + +from __future__ import annotations + +import torch +from diffusers.configuration_utils import register_to_config + +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + FlowMatchEulerDiscreteSchedulerOutput, +) + + +class DXPolicy: + def __init__( # noqa: PLR0913 + self, + denoising_output: torch.Tensor, + x_t_src: torch.Tensor, + sigma_t_src: torch.Tensor, + segment_size: float | torch.Tensor = 1.0, + shift: float = 1.0, + mode: str = "grid", + eps: float = 1e-4, + ) -> None: + self.x_t_src = x_t_src + self.ndim = x_t_src.dim() + self.shift = shift + self.eps = eps + if mode not in ("grid", "polynomial"): + raise ValueError(f"Unknown mode: {mode}") + self.mode = mode + + self.sigma_t_src = sigma_t_src.reshape( + *sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1]) + ) + self.raw_t_src = self._unwarp_t(self.sigma_t_src) + segment = segment_size + if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim(): + segment = segment.reshape( + *segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1]) + ) + self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0) + self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps) + self.denoising_output_x_0 = self._u_to_x_0( + denoising_output, self.x_t_src, self.sigma_t_src + ) + + @staticmethod + def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + n = x.size(1) + if n < 2: # noqa: PLR2004 + return x.squeeze(1) + t = t.clamp(min=0, max=1) * (n - 1) + t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2) + t1 = t0 + 1 + indices = torch.stack([t0, t1], dim=1) + values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:])) + return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1] + + def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor: + return sigma_t / (self.shift + (1 - self.shift) * sigma_t) + + @staticmethod + def _u_to_x_0( + denoising_output: torch.Tensor, + x_t: torch.Tensor, + sigma_t: torch.Tensor, + ) -> torch.Tensor: + return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output + + def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor: + sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1])) + raw_t = self._unwarp_t(sigma_t) + if self.mode == "grid": + x_0 = self._interpolate( + self.denoising_output_x_0, + (raw_t - self.raw_t_dst) / self.segment_size, + ) + else: + p_order = self.denoising_output_x_0.size(1) + diff_t = self.raw_t_src - raw_t + basis = torch.stack([diff_t**i for i in range(p_order)], dim=1) + x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1) + return (x_t - x_0) / sigma_t.clamp(min=self.eps) + + +def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor: + return shift * t / (1 + (shift - 1) * t) + + +def policy_rollout_fm( # noqa: PLR0913 + x_t_start: torch.Tensor, + sigma_t_start: torch.Tensor, + raw_t_start: torch.Tensor, + raw_t_end: torch.Tensor, + total_substeps: int, + policy: DXPolicy, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + num_batches = x_t_start.size(0) + ndim = x_t_start.dim() + shape = (num_batches, *((ndim - 1) * [1])) + raw_t_start = raw_t_start.reshape(shape) + raw_t_end = raw_t_end.reshape(shape) + sigma_t = sigma_t_start.reshape(shape) + + delta_raw_t = raw_t_start - raw_t_end + num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1) + substep_size = delta_raw_t / num_substeps + max_num_substeps = num_substeps.max() + + raw_t = raw_t_start + x_t = x_t_start + for substep_id in range(max_num_substeps.item()): + velocity = policy.pi(x_t, sigma_t) + raw_t_minus = (raw_t - substep_size).clamp(min=0) + sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift) + x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t) + + active_mask = num_substeps > substep_id + x_t = torch.where(active_mask, x_t_minus, x_t) + sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t) + raw_t = torch.where(active_mask, raw_t_minus, raw_t) + + return x_t, sigma_t, sigma_t.flatten() * 1_000 + + +class PiflowScheduler(FlowMatchEulerDiscreteScheduler): + is_piflow = True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 5.0, + n_grid: int = 10, + nfe: int | None = None, + eps: float = 1e-6, + final_step_size_scale: float = 0.5, + num_policy_substeps: int = 128, + ) -> None: + if n_grid < 2: + raise ValueError(f"PiflowScheduler requires n_grid >= 2, got {n_grid}") + if eps <= 0: + raise ValueError(f"PiflowScheduler requires eps > 0, got {eps}") + if not 0 < final_step_size_scale <= 1: + raise ValueError("PiflowScheduler requires 0 < final_step_size_scale <= 1") + if num_policy_substeps < 1: + raise ValueError("PiflowScheduler requires num_policy_substeps >= 1") + super().__init__( + num_train_timesteps=num_train_timesteps, + shift=shift, + ) + self.n_grid = int(n_grid) + self.eps = float(eps) + self.final_step_size_scale = float(final_step_size_scale) + self.num_policy_substeps = int(num_policy_substeps) + self._piflow_raw_timesteps = torch.empty(0) + + def set_timesteps( + self, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + sigmas: list[float] | None = None, + mu: float | None = None, + timesteps: list[float] | None = None, + ) -> None: + if sigmas is not None or mu is not None or timesteps is not None: + raise ValueError( + "PiflowScheduler only supports its configured distilled timestep schedule" + ) + if num_inference_steps is None or num_inference_steps < 1: + raise ValueError( + f"num_inference_steps must be positive, got {num_inference_steps}" + ) + one_minus_final = 1.0 - self.final_step_size_scale + segment = 1.0 / (num_inference_steps - one_minus_final) + raw = ( + 1.0 + - torch.arange(num_inference_steps, dtype=torch.float32, device=device) + * segment + ) + sigmas = shift_timesteps(raw, float(self.config.shift)) + self.num_inference_steps = int(num_inference_steps) + self._piflow_raw_timesteps = raw + self.timesteps = sigmas * self.config.num_train_timesteps + self.sigmas = torch.cat([sigmas, sigmas.new_zeros(1)]) + self._step_index = None + self._begin_index = None + + def _to_grid( + self, model_output: torch.Tensor, sample: torch.Tensor + ) -> torch.Tensor: + if ( + model_output.ndim != sample.ndim + or model_output.shape[:-1] != sample.shape[:-1] + ): + raise ValueError( + "Piflow model output must match sample shape except for the output channels: " + f"got {tuple(model_output.shape)} for sample {tuple(sample.shape)}" + ) + if model_output.shape[-1] % self.n_grid != 0: + raise ValueError( + f"Piflow model output channels {model_output.shape[-1]} are not divisible by n_grid={self.n_grid}" + ) + output_dim = model_output.shape[-1] // self.n_grid + if output_dim != sample.shape[-1]: + raise ValueError( + "Piflow model output channels do not match the sample: " + f"expected {sample.shape[-1] * self.n_grid}, got {model_output.shape[-1]}" + ) + return model_output.reshape( + *model_output.shape[:-1], self.n_grid, output_dim + ).movedim(-2, 1) + + def _policy_step( + self, model_output: torch.Tensor, sample: torch.Tensor, step_index: int + ) -> torch.Tensor: + model_output = self._to_grid(model_output, sample) + raw_src = self._piflow_raw_timesteps[step_index].to(device=sample.device) + raw_dst = ( + self._piflow_raw_timesteps[step_index + 1] + if step_index + 1 < self._piflow_raw_timesteps.numel() + else self._piflow_raw_timesteps.new_full((), self.eps) + ).to(device=sample.device) + sigma_src = self.sigmas[step_index].to(device=sample.device) + token_shape = (sample.shape[0], *((sample.ndim - 1) * [1])) + sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape) + segment = (raw_src - raw_dst).expand(sample.shape[0]) + policy = DXPolicy( + model_output, + sample, + sigma, + segment, + shift=float(self.config.shift), + mode="grid", + eps=self.eps, + ) + updated, _, _ = policy_rollout_fm( + sample, + sigma, + raw_src.expand(sample.shape[0]), + raw_dst.expand(sample.shape[0]), + self.num_policy_substeps, + policy, + ) + return updated.to(dtype=sample.dtype) + + def _step_index_for(self, timestep: torch.Tensor | float) -> int: + if self.step_index is None: + if self.begin_index is not None: + self._step_index = self.begin_index + else: + schedule_timesteps = self.timesteps + if not isinstance(schedule_timesteps, torch.Tensor): + schedule_timesteps = torch.as_tensor( + schedule_timesteps, dtype=torch.float32 + ) + timestep = torch.as_tensor( + timestep, + device=schedule_timesteps.device, + dtype=schedule_timesteps.dtype, + ) + indices = torch.nonzero(schedule_timesteps == timestep).flatten() + if not indices.numel(): + raise ValueError( + f"timestep {timestep.item()} is not in the Piflow schedule" + ) + position = 1 if indices.numel() > 1 else 0 + self._step_index = int(indices[position].item()) + if self.step_index is None or self.step_index >= self.num_inference_steps: + raise RuntimeError( + "PiflowScheduler.step called after the schedule was exhausted" + ) + return int(self.step_index) + + def step( + self, + model_output: torch.FloatTensor, + timestep: float | torch.FloatTensor, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> FlowMatchEulerDiscreteSchedulerOutput | tuple: + if isinstance(timestep, int) or isinstance( + timestep, (torch.IntTensor, torch.LongTensor) + ): + raise ValueError( + "Passing integer indices as timesteps to PiflowScheduler.step() is not supported; " + "pass a value from scheduler.timesteps instead" + ) + step_index = self._step_index_for(timestep) + # PiFlow's policy rollout performs its update in float32. Keep that + # precision across outer steps, matching the native sampler. + updated = self._policy_step(model_output, sample.to(torch.float32), step_index) + self._step_index += 1 + if return_dict: + return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=updated) + return (updated,) + + +EntryClass = PiflowScheduler diff --git a/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py b/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py new file mode 100644 index 0000000000..bbaed76bb3 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/upsampler/kandinsky6_sr_latent_upscaler.py @@ -0,0 +1,336 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 SR x2/x4 upsampler bank for scaled [B, C, T, H, W] KVAE latents. + +Norms are modulated by the source latent; temporal conv padding repeats edge frames. +_models entries follow checkpoint scale order and retain checkpoint module names.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + +from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import ( + Kandinsky6SRLatentUpscalerEntryConfig, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + + +class ReplicateTimeConv3d(nn.Conv3d): + """Conv3d with explicit edge-repeated temporal padding and zero spatial padding.""" + + def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None: + pad = kernel_size // 2 + super().__init__(in_channels, out_channels, kernel_size, padding=(0, pad, pad)) + self.temporal_pad = pad + + def forward(self, x: Tensor) -> Tensor: + x = F.pad( + x, (0, 0, 0, 0, self.temporal_pad, self.temporal_pad), mode="replicate" + ) + return super().forward(x) + + +class RMSNorm(nn.Module): + """Channel RMS norm of ``[B, C, T, H, W]`` features with a learnable per-channel gain.""" + + def __init__(self, dim: int) -> None: + super().__init__() + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1)) + + def forward(self, x: Tensor) -> Tensor: + # Upcast to float32 around the normalize (matches the Diffusers reference's + # Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the + # reference across the many ResidualBlock/X2Branch calls that route through this norm. + normalized = F.normalize(x.float(), dim=1).to(x.dtype) + return normalized * self.scale * self.gamma + + +class ModulatedRMSNorm(nn.Module): + """``RMSNorm(x) * conv_y(zq) + conv_b(zq)`` with 1x1x1 convs of the conditioning latent ``zq``.""" + + def __init__(self, dim: int, zq_dim: int) -> None: + super().__init__() + self.norm = RMSNorm(dim) + self.conv_y = nn.Conv3d(zq_dim, dim, kernel_size=1) + self.conv_b = nn.Conv3d(zq_dim, dim, kernel_size=1) + + def forward(self, x: Tensor, zq: Tensor) -> Tensor: + if zq.shape[2:] != x.shape[2:]: + zq = F.interpolate(zq, size=x.shape[2:], mode="nearest") + return self.norm(x) * self.conv_y(zq) + self.conv_b(zq) + + +class ResidualBlock(nn.Module): + """Pre-activation block ``norm1 -> SiLU -> conv1 -> norm2 -> SiLU -> conv2`` plus a (1x1x1 if + narrowing) skip.""" + + def __init__( + self, in_channels: int, out_channels: int, mid_channels: int, zq_dim: int + ) -> None: + super().__init__() + self.norm1 = ModulatedRMSNorm(in_channels, zq_dim) + self.conv1 = ReplicateTimeConv3d(in_channels, mid_channels, kernel_size=3) + self.norm2 = ModulatedRMSNorm(mid_channels, zq_dim) + self.conv2 = ReplicateTimeConv3d(mid_channels, out_channels, kernel_size=3) + self.shortcut: nn.Module = ( + nn.Identity() + if in_channels == out_channels + else ReplicateTimeConv3d(in_channels, out_channels, kernel_size=1) + ) + + def forward(self, x: Tensor, zq: Tensor) -> Tensor: + h = F.silu(self.norm1(x, zq)) + h = self.conv1(h) + h = F.silu(self.norm2(h, zq)) + h = self.conv2(h) + return self.shortcut(x) + h + + +def nearest_2x(x: Tensor) -> Tensor: + """Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch).""" + b, c, t, h, w = x.shape + x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + x = F.interpolate(x, scale_factor=2, mode="nearest") + return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4) + + +class PXSUpsample(nn.Module): + """2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``.""" + + def __init__(self, channels: int) -> None: + super().__init__() + self.spatial_conv = nn.Conv3d( + channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1) + ) + self.linear = nn.Conv3d(channels, channels, kernel_size=1) + + def forward(self, x: Tensor) -> Tensor: + up = nearest_2x(x) + return self.linear(up + self.spatial_conv(up)) + + +class X2Finisher(nn.Module): + """The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize).""" + + def __init__(self, channels: int) -> None: + super().__init__() + self.spatial_conv = nn.Conv3d( + channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1) + ) + self.linear = nn.Conv3d(channels, channels, kernel_size=1) + + def forward(self, x: Tensor) -> Tensor: + return self.linear(x + self.spatial_conv(x)) + + +def _stem(in_channels: int, out_channels: int) -> nn.Sequential: + return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3)) + + +def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential: + """Unused training head whose checkpoint layout remains positional.""" + return nn.Sequential( + RMSNorm(channels), + nn.SiLU(), + ReplicateTimeConv3d(channels, out_channels, kernel_size=3), + ) + + +class Kandinsky6SRLatentUpscalerOutputHead(nn.Module): + """Named ``norm -> activation -> conv`` projection used by current checkpoints.""" + + def __init__(self, channels: int, out_channels: int, zq_dim: int) -> None: + super().__init__() + self.norm = ModulatedRMSNorm(channels, zq_dim) + self.activation = nn.SiLU() + self.conv = ReplicateTimeConv3d(channels, out_channels, kernel_size=3) + + def forward(self, x: Tensor, zq: Tensor) -> Tensor: + return self.conv(self.activation(self.norm(x, zq))) + + +def _residual_stack( + count: int, in_channels: int, channels: int, expand_ratio: int, zq_dim: int +) -> nn.Sequential: + """``count`` blocks at width ``channels``; the first one narrows from ``in_channels``.""" + return nn.Sequential( + *( + ResidualBlock( + in_channels if index == 0 else channels, + channels, + channels * expand_ratio, + zq_dim, + ) + for index in range(count) + ) + ) + + +def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor: + for block in blocks: + x = block(x, zq) + return x + + +def _second_stage( + upsample: nn.Module, + blocks: nn.Sequential, + head: Kandinsky6SRLatentUpscalerOutputHead, + x: Tensor, + zq: Tensor, +) -> Tensor: + """``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``.""" + x = _run_blocks(blocks, upsample(x), zq) + return head(x, zq) + + +class X2Branch(nn.Module): + """Weights exclusive to the x2 path.""" + + def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None: + super().__init__() + c = config.in_channels + w1, w2, w3 = config.stage_channels + er = config.expand_ratio + self.adapter = _residual_stack(config.x2_adapter_blocks, w1, w1, er, c) + self.finisher = X2Finisher(w1) + self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c) + self.upsample = PXSUpsample(w2) + self.blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c) + self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c) + + def forward(self, x: Tensor, zq: Tensor) -> Tensor: + x = _run_blocks(self.adapter, x, zq) + x = self.finisher(x) + x = _run_blocks(self.mid_blocks, x, zq) + return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq) + + +class Kandinsky6SRLatentUpscaler(nn.Module): + """One bank entry: ``[B, C, T, h, w]`` -> ``[B, C, T, s*h, s*w]`` for ``s`` = 4, or 2 with + ``enable_x2_entry``.""" + + def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None: + super().__init__() + c = config.in_channels + w1, w2, w3 = config.stage_channels + er = config.expand_ratio + self.input_proj = _stem(c, w1) + # The 2x deep-supervision head of training: part of the checkpoint, never run at inference. + self.mid_output_head = _mid_output_head(w2, c) + self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c) + self.upsample_1 = PXSUpsample(w1) + self.upsample_2 = PXSUpsample(w2) + self.pre_blocks = _residual_stack(config.num_pre_blocks, w1, w1, er, c) + self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c) + self.post_blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c) + self.mid_input_proj: nn.Sequential | None = None + self.x2_branch: X2Branch | None = None + if config.enable_x2_entry: + self.mid_input_proj = _stem(c, config.hidden_channels) + self.x2_branch = X2Branch(config) + + def forward(self, z: Tensor, scale: int) -> Tensor: + zq = z + if scale == 2: + if self.x2_branch is None or self.mid_input_proj is None: + raise ValueError( + "this latent-upscaler entry has no x2 path (enable_x2_entry=false)" + ) + return self.x2_branch(self.mid_input_proj(z), zq) + if scale != 4: + raise ValueError( + f"latent-upscaler entries upsample by 2 or 4, got scale={scale!r}" + ) + x = _run_blocks(self.pre_blocks, self.input_proj(z), zq) + x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq) + return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq) + + +class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin): + """Select the checkpoint bank entry to upscale scaled KVAE latents in H/W.""" + + layerwise_offload_dit_group_enabled = False + + def __init__( + self, + models: Sequence[Mapping[str, Any]], + scaling_factor: float = 1.0, + scales: Sequence[int] = (2, 4), + ) -> None: + super().__init__() + if not models: + raise ValueError("latent upscaler config declares no `models` entries") + entries = [ + Kandinsky6SRLatentUpscalerEntryConfig.from_dict(spec, index) + for index, spec in enumerate(models) + ] + entries_by_scale = {entry.target_scale: entry for entry in entries} + if len(entries_by_scale) != len(entries): + raise ValueError("latent upscaler `models` has duplicate target scales") + self._scales = tuple(int(scale) for scale in scales) + if len(set(self._scales)) != len(self._scales) or set(self._scales) != set( + entries_by_scale + ): + raise ValueError( + "latent upscaler `scales` must match the target scales in `models`: " + f"{self._scales} vs {tuple(entries_by_scale)}" + ) + self.scaling_factor = float(scaling_factor) + self._models = nn.ModuleList( + [ + Kandinsky6SRLatentUpscaler(entries_by_scale[scale]) + for scale in self._scales + ] + ) + + self.layer_names = [] + for index, entry in enumerate(self._models): + self.layer_names += [ + f"_models.{index}.pre_blocks", + f"_models.{index}.mid_blocks", + f"_models.{index}.post_blocks", + ] + if entry.x2_branch is not None: + self.layer_names += [ + f"_models.{index}.x2_branch.adapter", + f"_models.{index}.x2_branch.mid_blocks", + f"_models.{index}.x2_branch.blocks", + ] + + @property + def scales(self) -> tuple[int, ...]: + """The upscale factors this bank serves, ascending.""" + return tuple(sorted(self._scales)) + + def for_scale(self, scale: int) -> nn.Module | None: + """Return the bank entry matching ``scale``, or ``None`` if the bank does not serve it.""" + scale = int(scale) + return ( + self._models[self._scales.index(scale)] if scale in self._scales else None + ) + + def upscale(self, latent: Tensor, *, scale: int) -> Tensor: + """Run the entry for ``scale`` on a *scaled* latent ``[B, C, T, h, w]``.""" + entry = self.for_scale(scale) + if entry is None: + raise ValueError( + f"the latent-upscaler bank has no x{scale} entry (scales: {self.scales})" + ) + return entry(latent, scale) + + def forward(self, z: Tensor, *, scale: int) -> Tensor: + return self.upscale(z, scale=scale) + + +EntryClass = Kandinsky6SRLatentUpscalerBank diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/alias_free.py b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py similarity index 88% rename from python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/alias_free.py rename to python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py index de8c266f11..b45405c1a2 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/alias_free.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/alias_free.py @@ -1,5 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0 +"""Alias-free-resampling DSP primitives shared by the BigVGAN-lineage vocoders. + +Generic, architecture-agnostic building blocks (Kaiser-windowed-sinc low-pass +filtering, 2x up/down resampling, and the up-activate-down ``Activation1d`` +wrapper) -- not specific to any one model. Previously duplicated +byte-for-byte between ``kandinsky6_audio_vae/alias_free.py`` and +``minimax_h3_audio_vae/alias_free.py``; both ``BigVGANV2``/``BigVGAN`` +implementations now import ``Activation1d`` from here instead, matching this +package's existing cross-model-shared-VAE-code convention (see ``common.py`` +in this same directory, reused by ``dac.py``, ``ltx_2_audio.py``, +``wanvae.py``, ``hunyuanvae.py``, and others). +""" import math diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/python/sglang/multimodal_gen/runtime/models/vaes/common.py index 851f4ef9fb..bc31fc4023 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/common.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/common.py @@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.cache.conditioning import ( from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_group_coordinator, get_decode_parallel_world_size, + get_sp_group, get_sp_parallel_rank, get_sp_world_size, model_parallel_is_initialized, @@ -286,7 +287,8 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): """ Parallel version of tiled_decode that distributes both temporal and spatial computation across GPUs """ - world_size, rank = get_sp_world_size(), get_sp_parallel_rank() + sp_group = get_sp_group() + world_size, rank = sp_group.world_size, sp_group.rank_in_group _, _, T, H, W = z.shape tile_latent_min_height = ( @@ -361,7 +363,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): torch.zeros(1, device=results.device, dtype=torch.int64) for _ in range(world_size) ] - dist.all_gather(all_sizes, local_size) + dist.all_gather(all_sizes, local_size, group=sp_group.device_group) max_size = max(size.item() for size in all_sizes) padded_results = torch.zeros( @@ -375,8 +377,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): .repeat(world_size, *[1] * len(padded_results.shape)) .contiguous() ) - all_gather_single(gathered_results, padded_results) - dist.all_gather_object(gathered_dim_metadata, local_dim_metadata) + all_gather_single( + gathered_results.view(-1), padded_results, group=sp_group.device_group + ) + dist.all_gather_object( + gathered_dim_metadata, local_dim_metadata, group=sp_group.cpu_group + ) gathered_dim_metadata = cast(list[list[torch.Size]], gathered_dim_metadata) data: list = [ diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py index 93d424f71f..1c91516a79 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/hunyuanvae.py @@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.layers.parallel_conv import ( disable_spatial_parallel_decode, gather_and_trim_height, gather_variable_height, + spatial_parallel_decode_disabled, split_height_for_parallel_decode, ) from sglang.multimodal_gen.runtime.models.vaes.common import ( @@ -125,7 +126,11 @@ class HunyuanVAEAttention(nn.Module): ) def forward( - self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + *, + num_frames: int | None = None, ) -> torch.Tensor: residual = hidden_states @@ -145,10 +150,30 @@ class HunyuanVAEAttention(nn.Module): key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2) - # Perform scaled dot-product attention - hidden_states = F.scaled_dot_product_attention( - query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False - ) + if num_frames is None: + hidden_states = F.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + is_causal=False, + ) + else: + # each frame sees all spatial tokens in itself and earlier frames + # slicing the key prefix avoids a quadratic whole-video mask + tokens_per_frame = sequence_length // num_frames + hidden_states = torch.cat( + [ + F.scaled_dot_product_attention( + query[:, :, start : start + tokens_per_frame], + key[:, :, : start + tokens_per_frame], + value[:, :, : start + tokens_per_frame], + ) + for start in range(0, sequence_length, tokens_per_frame) + ], + dim=2, + ) # Reshape back hidden_states = hidden_states.transpose(1, 2).reshape( @@ -278,6 +303,8 @@ class HunyuanVideoUpsampleCausal3D(nn.Module): else: hidden_states = first_frame + # release interpolation buffers before allocating convolution workspace + del first_frame, other_frames hidden_states = self.conv(hidden_states) return hidden_states @@ -422,19 +449,25 @@ class HunyuanVideoMidBlock3D(nn.Module): self, attn: HunyuanVAEAttention, hidden_states: torch.Tensor ) -> torch.Tensor: heights = None - if self.spatial_parallel: + spatial_parallel = ( + self.spatial_parallel and not spatial_parallel_decode_disabled() + ) + if spatial_parallel: hidden_states, heights = gather_variable_height(hidden_states) batch_size, num_channels, num_frames, height, width = hidden_states.shape hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3) - attention_mask = prepare_causal_attention_mask( - num_frames, - height * width, - hidden_states.dtype, - hidden_states.device, - batch_size=batch_size, - ) - hidden_states = attn(hidden_states, attention_mask=attention_mask) + if spatial_parallel: + hidden_states = attn(hidden_states, num_frames=num_frames) + else: + attention_mask = prepare_causal_attention_mask( + num_frames, + height * width, + hidden_states.dtype, + hidden_states.device, + batch_size=batch_size, + ) + hidden_states = attn(hidden_states, attention_mask=attention_mask) hidden_states = hidden_states.unflatten(1, (num_frames, height, width)).permute( 0, 4, 1, 2, 3 ) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py new file mode 100644 index 0000000000..aa8ed2bfb3 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 mel VAE and waveform vocoder. + +The vae.*, vocoder.* and mel_converter.* names match the official checkpoint, +allowing the generic VAELoader to load the full tree without key remapping.""" + +from __future__ import annotations + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAEConfig, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.bigvgan import ( + BigVGANV2, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mel_converter import ( + MelConverter, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import ( + MMAudioVAE, +) + + +class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin): + """MMAudio mel codec and BigVGAN vocoder with checkpoint-compatible nesting. + + decode: [B, embed_dim, A] -> [B, num_mels, T] + vocode: [B, num_mels, T] -> [B, 1, samples] + wrapped_decode and wrapped_encode compose the codec and waveform stages.""" + + # official checkpoint class name + _aliases = ["MMAudioVAE"] + + # Decode-only small model relative to the DiT; no benefit from DiT-group + # offload tuning knobs. Mirrors MiniMaxH3AudioVAE's choice. + layerwise_offload_dit_group_enabled = False + + def __init__(self, config: Kandinsky6AudioVAEConfig) -> None: + super().__init__() + arch = config.arch_config + self.scaling_factor = float(arch.scaling_factor) + self.mean_value = float(arch.mean_value) + # audio latent stride; frame-count alignment is owned by the pipeline config + self.downsample_factor = 1024 + + self.mel_converter = MelConverter( + sampling_rate=44100, + n_fft=2048, + num_mels=128, + hop_size=512, + win_size=2048, + fmin=0, + fmax=22050, + ) + # checkpoint weights include the encoder regardless of need_vae_encoder + self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True) + # normalize random initialization before loading; checkpoint weights are + # already normalized and must never be renormalized + self.vae.remove_weight_norm() + self.vocoder = BigVGANV2(dict(arch.vocoder_config)) + # strip PyTorch weight_norm before loading plain .weight checkpoint keys; + # this differs from MMAudio's RMS renormalization above + self.vocoder.remove_weight_norm() + + # resolution containers are not called; hooks belong on their blocks + self.layer_names = ( + [ + f"{path}.{index}.{group}" + for path, levels in ( + ("vae.encoder.down", self.vae.encoder.down), + ("vae.decoder.up", self.vae.decoder.up), + ) + for index in range(len(levels)) + for group in ("block", "attn") + ] + + [f"vocoder.ups.{index}" for index in range(len(self.vocoder.ups))] + + ["vocoder.resblocks"] + ) + + def decode(self, latents: torch.Tensor) -> torch.Tensor: + """latents: [B, embed_dim, A] (1D-conv channel-first) -> mel: [B, num_mels, T].""" + return self.vae.decode(latents, unnormalize_output=True) + + def vocode(self, mel: torch.Tensor) -> torch.Tensor: + """mel: [B, num_mels, T] -> waveform: [B, 1, samples].""" + # mel denormalization returns fp32; cast to the loaded vocoder's dtype + vocoder_dtype = next(self.vocoder.parameters()).dtype + return self.vocoder(mel.to(dtype=vocoder_dtype)) + + def wrapped_decode(self, latents: torch.Tensor) -> torch.Tensor: + """latents -> waveform in one call, matching the diffusers + reference's MMAudioVAE.wrapped_decode.""" + return self.vocode(self.decode(latents)) + + def encode_audio(self, waveform: torch.Tensor): + """waveform -> mel -> VAE posterior. See MelConverter's module + docstring: not exercised by / verified against the current + (decode-only) inference path.""" + mel = self.mel_converter(waveform) + return self.vae.encode(mel) + + def wrapped_encode(self, waveform: torch.Tensor) -> torch.Tensor: + """waveform -> mean audio latent in one call, matching the + diffusers reference's MMAudioVAE.wrapped_encode.""" + return self.encode_audio(waveform).mean + + def remove_weight_norm(self) -> Kandinsky6AudioVAE: + """Idempotent MMAudio weight normalization, normally applied before loading.""" + self.vae.remove_weight_norm() + return self + + +EntryClass = Kandinsky6AudioVAE + +__all__ = ["Kandinsky6AudioVAE"] diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/THIRD_PARTY_LICENSES b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/THIRD_PARTY_LICENSES new file mode 100644 index 0000000000..15eaffae1b --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/THIRD_PARTY_LICENSES @@ -0,0 +1,31 @@ +This directory contains code adapted from the following MIT-licensed projects: + +- NVIDIA BigVGAN, bigvgan.py + https://github.com/NVIDIA/BigVGAN + Copyright (c) 2024 NVIDIA CORPORATION. + +- MMAudio, mmaudio_vae.py + https://github.com/hkchengrex/MMAudio + Copyright (c) 2024 Sony Research Inc. + +The following license applies to those portions of the code: + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/__init__.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/__init__.py new file mode 100644 index 0000000000..508ca64e6a --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Audio codec building blocks; the registry entry lives in ../kandinsky6_audio.py.""" + +from .bigvgan import BigVGANV2 +from .mel_converter import MelConverter +from .mmaudio_vae import DiagonalGaussianDistribution, MMAudioVAE + +__all__ = ["BigVGANV2", "DiagonalGaussianDistribution", "MMAudioVAE", "MelConverter"] diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py new file mode 100644 index 0000000000..cab3d2ff12 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/bigvgan.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 NVIDIA CORPORATION. +# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's +# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is +# intentionally excluded; this module uses the portable PyTorch path for +# deterministic loading and parity across platforms. +# Original license terms are retained in THIRD_PARTY_LICENSES. +# +# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/ +# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than +# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` -- +# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and +# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more), +# while this implementation supports either resblock variant or activation. The +# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/ +# Activation1d) ARE architecture-agnostic DSP building blocks shared with +# that same sibling module, though -- both import ``Activation1d`` from +# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside +# ``common.py``) instead of each keeping its own copy. + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn as nn +from torch.nn import Conv1d, ConvTranspose1d +from torch.nn.utils.parametrizations import weight_norm +from torch.nn.utils.parametrize import remove_parametrizations + +from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d + + +class AttrDict(dict): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.__dict__ = self + + +def get_padding(kernel_size: int, dilation: int = 1) -> int: + return int((kernel_size * dilation - dilation) / 2) + + +def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None: + if "Conv" in module.__class__.__name__: + module.weight.data.normal_(mean, std) + + +class Snake(nn.Module): + def __init__( + self, + in_features: int, + alpha: float = 1.0, + alpha_trainable: bool = True, + alpha_logscale: bool = False, + ) -> None: + super().__init__() + self.in_features = in_features + self.alpha_logscale = alpha_logscale + initial = ( + torch.zeros(in_features) * alpha + if alpha_logscale + else torch.ones(in_features) * alpha + ) + self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable) + self.no_div_by_zero = 1e-9 + + def forward(self, x: torch.Tensor) -> torch.Tensor: + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) + if self.alpha_logscale: + alpha = torch.exp(alpha) + return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow( + torch.sin(x * alpha), 2 + ) + + +class SnakeBeta(nn.Module): + def __init__( + self, + in_features: int, + alpha: float = 1.0, + alpha_trainable: bool = True, + alpha_logscale: bool = False, + ) -> None: + super().__init__() + self.in_features = in_features + self.alpha_logscale = alpha_logscale + initial = ( + torch.zeros(in_features) * alpha + if alpha_logscale + else torch.ones(in_features) * alpha + ) + self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable) + self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable) + self.no_div_by_zero = 1e-9 + + def forward(self, x: torch.Tensor) -> torch.Tensor: + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) + beta = self.beta.unsqueeze(0).unsqueeze(-1) + if self.alpha_logscale: + alpha = torch.exp(alpha) + beta = torch.exp(beta) + return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow( + torch.sin(x * alpha), 2 + ) + + +def _activation(name: str, channels: int, logscale: bool) -> Activation1d: + if name == "snake": + activation = Snake(channels, alpha_logscale=logscale) + elif name == "snakebeta": + activation = SnakeBeta(channels, alpha_logscale=logscale) + else: + raise ValueError(f"Unsupported BigVGAN activation: {name}") + return Activation1d(activation) + + +class AMPBlock1(nn.Module): + def __init__( + self, + config: AttrDict, + channels: int, + kernel_size: int = 3, + dilation: tuple[int, ...] = (1, 3, 5), + activation: str = "snake", + ) -> None: + super().__init__() + self.convs1 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + stride=1, + dilation=rate, + padding=get_padding(kernel_size, rate), + ) + ) + for rate in dilation + ] + ) + self.convs1.apply(init_weights) + self.convs2 = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + stride=1, + dilation=1, + padding=get_padding(kernel_size, 1), + ) + ) + for _ in dilation + ] + ) + self.convs2.apply(init_weights) + self.activations = nn.ModuleList( + [ + _activation(activation, channels, config.snake_logscale) + for _ in range(len(self.convs1) + len(self.convs2)) + ] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + acts1, acts2 = self.activations[::2], self.activations[1::2] + for conv1, conv2, act1, act2 in zip( + self.convs1, self.convs2, acts1, acts2, strict=True + ): + residual = conv2(act2(conv1(act1(x)))) + x = residual + x + return x + + def remove_weight_norm(self) -> None: + for layer in self.convs1: + remove_parametrizations(layer, "weight") + for layer in self.convs2: + remove_parametrizations(layer, "weight") + + +class AMPBlock2(nn.Module): + def __init__( + self, + config: AttrDict, + channels: int, + kernel_size: int = 3, + dilation: tuple[int, ...] = (1, 3, 5), + activation: str = "snake", + ) -> None: + super().__init__() + self.convs = nn.ModuleList( + [ + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + stride=1, + dilation=rate, + padding=get_padding(kernel_size, rate), + ) + ) + for rate in dilation + ] + ) + self.convs.apply(init_weights) + self.activations = nn.ModuleList( + [ + _activation(activation, channels, config.snake_logscale) + for _ in self.convs + ] + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + for conv, activation in zip(self.convs, self.activations, strict=True): + x = conv(activation(x)) + x + return x + + def remove_weight_norm(self) -> None: + for layer in self.convs: + remove_parametrizations(layer, "weight") + + +class BigVGANV2(nn.Module): + """BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path.""" + + def __init__(self, config: dict[str, Any]) -> None: + super().__init__() + config = dict(config) + config.pop("_class_name", None) + config.pop("architectures", None) + weight_norm_removed = bool(config.pop("weight_norm_removed", False)) + self.config = AttrDict(config) + if self.config.get("use_cuda_kernel", False): + raise ValueError( + "Kandinsky6 BigVGANV2 supports only the portable PyTorch path" + ) + self.config["use_cuda_kernel"] = False + self.num_kernels = len(self.config.resblock_kernel_sizes) + self.num_upsamples = len(self.config.upsample_rates) + self.conv_pre = weight_norm( + Conv1d( + self.config.num_mels, + self.config.upsample_initial_channel, + 7, + 1, + padding=3, + ) + ) + if self.config.resblock == "1": + block_class = AMPBlock1 + elif self.config.resblock == "2": + block_class = AMPBlock2 + else: + raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}") + + self.ups = nn.ModuleList() + for index, (rate, kernel) in enumerate( + zip( + self.config.upsample_rates, + self.config.upsample_kernel_sizes, + strict=True, + ) + ): + self.ups.append( + nn.ModuleList( + [ + weight_norm( + ConvTranspose1d( + self.config.upsample_initial_channel // (2**index), + self.config.upsample_initial_channel + // (2 ** (index + 1)), + kernel, + rate, + padding=(kernel - rate) // 2, + ) + ) + ] + ) + ) + + self.resblocks = nn.ModuleList() + for index in range(len(self.ups)): + channels = self.config.upsample_initial_channel // (2 ** (index + 1)) + for kernel, dilation in zip( + self.config.resblock_kernel_sizes, + self.config.resblock_dilation_sizes, + strict=True, + ): + self.resblocks.append( + block_class( + self.config, + channels, + kernel, + tuple(dilation), + activation=self.config.activation, + ) + ) + + channels = self.config.upsample_initial_channel // (2 ** len(self.ups)) + self.activation_post = _activation( + self.config.activation, channels, self.config.snake_logscale + ) + self.use_bias_at_final = self.config.get("use_bias_at_final", True) + self.conv_post = weight_norm( + Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final) + ) + for upsampler in self.ups: + upsampler.apply(init_weights) + self.conv_post.apply(init_weights) + self.use_tanh_at_final = self.config.get("use_tanh_at_final", True) + if weight_norm_removed: + self.remove_weight_norm() + + def forward(self, mel: torch.Tensor) -> torch.Tensor: + hidden = self.conv_pre(mel) + for index in range(self.num_upsamples): + for upsampler in self.ups[index]: + hidden = upsampler(hidden) + accumulated = None + for kernel in range(self.num_kernels): + block_output = self.resblocks[index * self.num_kernels + kernel](hidden) + if accumulated is None: + accumulated = block_output + else: + # Preserve BigVGAN's published sequential accumulation + # order. Tree reduction drifts through later nonlinear + # upsampling stages with the full checkpoint. + accumulated += block_output + assert accumulated is not None + hidden = accumulated / self.num_kernels + hidden = self.conv_post(self.activation_post(hidden)) + if self.use_tanh_at_final: + return torch.tanh(hidden) + return torch.clamp(hidden, min=-1.0, max=1.0) + + def remove_weight_norm(self) -> None: + try: + for upsamplers in self.ups: + for upsampler in upsamplers: + remove_parametrizations(upsampler, "weight") + for block in self.resblocks: + block.remove_weight_norm() + remove_parametrizations(self.conv_pre, "weight") + remove_parametrizations(self.conv_post, "weight") + except ValueError: + # Idempotent for pipeline setup and converted checkpoints. + return diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mel_converter.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mel_converter.py new file mode 100644 index 0000000000..4ca4f9f1de --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mel_converter.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from FastVideo's fastvideo/models/audio/kandinsky6_audio_vae.py. +"""Checkpoint-compatible waveform-to-log-mel frontend. + +Included for state-dict completeness; generation only decodes audio latents, +so waveform encoding has not been validated end-to-end.""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class MelConverter(nn.Module): + """Waveform -> log-mel-spectrogram STFT frontend. See module docstring: + not exercised by the current (decode-only) inference path.""" + + def __init__( + self, + *, + sampling_rate: int = 44100, + n_fft: int = 2048, + num_mels: int = 128, + hop_size: int = 512, + win_size: int = 2048, + fmin: int = 0, + fmax: int | None = 22050, + ) -> None: + super().__init__() + self.n_fft = n_fft + self.hop_size = hop_size + self.win_size = win_size + self.register_buffer( + "hann_window", torch.hann_window(win_size), persistent=True + ) + try: + from librosa.filters import mel as librosa_mel_fn + + mel_basis = torch.from_numpy( + librosa_mel_fn( + sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax + ) + ).float() + except Exception: + # generation does not use this buffer; the checkpoint supplies its real values + mel_basis = torch.zeros(num_mels, n_fft // 2 + 1) + self.register_buffer("mel_basis", mel_basis, persistent=True) + + def forward(self, waveform: torch.Tensor, center: bool = False) -> torch.Tensor: + waveform = waveform.clamp(min=-1.0, max=1.0) + pad = (self.n_fft - self.hop_size) // 2 + padded = F.pad(waveform.unsqueeze(1), (pad, pad), mode="reflect").squeeze(1) + spec = torch.stft( + padded, + self.n_fft, + hop_length=self.hop_size, + win_length=self.win_size, + window=self.hann_window, + center=center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + spec = torch.view_as_real(spec) + magnitude = torch.sqrt(spec.pow(2).sum(-1) + 1e-9).float() + mel = torch.matmul(self.mel_basis, magnitude) + return torch.log(torch.clamp(mel, min=1e-5)) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py new file mode 100644 index 0000000000..17276fa45a --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_audio_vae/mmaudio_vae.py @@ -0,0 +1,854 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License, +# Copyright (c) 2024 Sony Research Inc., via FastVideo's +# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram +# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae`` +# (see kandinsky6_audio.py in the parent directory). +# Original license terms are retained in THIRD_PARTY_LICENSES. +# +# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its +# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is +# dropped -- sglang's generic VAELoader never calls a per-submodule +# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict`` +# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code +# here. +"""Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec).""" + +from __future__ import annotations + +import logging +import math +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +logger = logging.getLogger(__name__) + + +DATA_MEAN_80D = [ + -1.6058, + -1.3676, + -1.2520, + -1.2453, + -1.2078, + -1.2224, + -1.2419, + -1.2439, + -1.2922, + -1.2927, + -1.3170, + -1.3543, + -1.3401, + -1.3836, + -1.3907, + -1.3912, + -1.4313, + -1.4152, + -1.4527, + -1.4728, + -1.4568, + -1.5101, + -1.5051, + -1.5172, + -1.5623, + -1.5373, + -1.5746, + -1.5687, + -1.6032, + -1.6131, + -1.6081, + -1.6331, + -1.6489, + -1.6489, + -1.6700, + -1.6738, + -1.6953, + -1.6969, + -1.7048, + -1.7280, + -1.7361, + -1.7495, + -1.7658, + -1.7814, + -1.7889, + -1.8064, + -1.8221, + -1.8377, + -1.8417, + -1.8643, + -1.8857, + -1.8929, + -1.9173, + -1.9379, + -1.9531, + -1.9673, + -1.9824, + -2.0042, + -2.0215, + -2.0436, + -2.0766, + -2.1064, + -2.1418, + -2.1855, + -2.2319, + -2.2767, + -2.3161, + -2.3572, + -2.3954, + -2.4282, + -2.4659, + -2.5072, + -2.5552, + -2.6074, + -2.6584, + -2.7107, + -2.7634, + -2.8266, + -2.8981, + -2.9673, +] + +DATA_STD_80D = [ + 1.0291, + 1.0411, + 1.0043, + 0.9820, + 0.9677, + 0.9543, + 0.9450, + 0.9392, + 0.9343, + 0.9297, + 0.9276, + 0.9263, + 0.9242, + 0.9254, + 0.9232, + 0.9281, + 0.9263, + 0.9315, + 0.9274, + 0.9247, + 0.9277, + 0.9199, + 0.9188, + 0.9194, + 0.9160, + 0.9161, + 0.9146, + 0.9161, + 0.9100, + 0.9095, + 0.9145, + 0.9076, + 0.9066, + 0.9095, + 0.9032, + 0.9043, + 0.9038, + 0.9011, + 0.9019, + 0.9010, + 0.8984, + 0.8983, + 0.8986, + 0.8961, + 0.8962, + 0.8978, + 0.8962, + 0.8973, + 0.8993, + 0.8976, + 0.8995, + 0.9016, + 0.8982, + 0.8972, + 0.8974, + 0.8949, + 0.8940, + 0.8947, + 0.8936, + 0.8939, + 0.8951, + 0.8956, + 0.9017, + 0.9167, + 0.9436, + 0.9690, + 1.0003, + 1.0225, + 1.0381, + 1.0491, + 1.0545, + 1.0604, + 1.0761, + 1.0929, + 1.1089, + 1.1196, + 1.1176, + 1.1156, + 1.1117, + 1.1070, +] + +DATA_MEAN_128D = [ + -3.3462, + -2.6723, + -2.4893, + -2.3143, + -2.2664, + -2.3317, + -2.1802, + -2.4006, + -2.2357, + -2.4597, + -2.3717, + -2.4690, + -2.5142, + -2.4919, + -2.6610, + -2.5047, + -2.7483, + -2.5926, + -2.7462, + -2.7033, + -2.7386, + -2.8112, + -2.7502, + -2.9594, + -2.7473, + -3.0035, + -2.8891, + -2.9922, + -2.9856, + -3.0157, + -3.1191, + -2.9893, + -3.1718, + -3.0745, + -3.1879, + -3.2310, + -3.1424, + -3.2296, + -3.2791, + -3.2782, + -3.2756, + -3.3134, + -3.3509, + -3.3750, + -3.3951, + -3.3698, + -3.4505, + -3.4509, + -3.5089, + -3.4647, + -3.5536, + -3.5788, + -3.5867, + -3.6036, + -3.6400, + -3.6747, + -3.7072, + -3.7279, + -3.7283, + -3.7795, + -3.8259, + -3.8447, + -3.8663, + -3.9182, + -3.9605, + -3.9861, + -4.0105, + -4.0373, + -4.0762, + -4.1121, + -4.1488, + -4.1874, + -4.2461, + -4.3170, + -4.3639, + -4.4452, + -4.5282, + -4.6297, + -4.7019, + -4.7960, + -4.8700, + -4.9507, + -5.0303, + -5.0866, + -5.1634, + -5.2342, + -5.3242, + -5.4053, + -5.4927, + -5.5712, + -5.6464, + -5.7052, + -5.7619, + -5.8410, + -5.9188, + -6.0103, + -6.0955, + -6.1673, + -6.2362, + -6.3120, + -6.3926, + -6.4797, + -6.5565, + -6.6511, + -6.8130, + -6.9961, + -7.1275, + -7.2457, + -7.3576, + -7.4663, + -7.6136, + -7.7469, + -7.8815, + -8.0132, + -8.1515, + -8.3071, + -8.4722, + -8.7418, + -9.3975, + -9.6628, + -9.7671, + -9.8863, + -9.9992, + -10.0860, + -10.1709, + -10.5418, + -11.2795, + -11.3861, +] + +DATA_STD_128D = [ + 2.3804, + 2.4368, + 2.3772, + 2.3145, + 2.2803, + 2.2510, + 2.2316, + 2.2083, + 2.1996, + 2.1835, + 2.1769, + 2.1659, + 2.1631, + 2.1618, + 2.1540, + 2.1606, + 2.1571, + 2.1567, + 2.1612, + 2.1579, + 2.1679, + 2.1683, + 2.1634, + 2.1557, + 2.1668, + 2.1518, + 2.1415, + 2.1449, + 2.1406, + 2.1350, + 2.1313, + 2.1415, + 2.1281, + 2.1352, + 2.1219, + 2.1182, + 2.1327, + 2.1195, + 2.1137, + 2.1080, + 2.1179, + 2.1036, + 2.1087, + 2.1036, + 2.1015, + 2.1068, + 2.0975, + 2.0991, + 2.0902, + 2.1015, + 2.0857, + 2.0920, + 2.0893, + 2.0897, + 2.0910, + 2.0881, + 2.0925, + 2.0873, + 2.0960, + 2.0900, + 2.0957, + 2.0958, + 2.0978, + 2.0936, + 2.0886, + 2.0905, + 2.0845, + 2.0855, + 2.0796, + 2.0840, + 2.0813, + 2.0817, + 2.0838, + 2.0840, + 2.0917, + 2.1061, + 2.1431, + 2.1976, + 2.2482, + 2.3055, + 2.3700, + 2.4088, + 2.4372, + 2.4609, + 2.4731, + 2.4847, + 2.5072, + 2.5451, + 2.5772, + 2.6147, + 2.6529, + 2.6596, + 2.6645, + 2.6726, + 2.6803, + 2.6812, + 2.6899, + 2.6916, + 2.6931, + 2.6998, + 2.7062, + 2.7262, + 2.7222, + 2.7158, + 2.7041, + 2.7485, + 2.7491, + 2.7451, + 2.7485, + 2.7233, + 2.7297, + 2.7233, + 2.7145, + 2.6958, + 2.6788, + 2.6439, + 2.6007, + 2.4786, + 2.2469, + 2.1877, + 2.1392, + 2.0717, + 2.0107, + 1.9676, + 1.9140, + 1.7102, + 0.9101, + 0.7164, +] + + +_NORMALIZATION_EPSILON = 1e-4 +_SILU_DIVISOR = 0.596 +_RESIDUAL_BLEND = 0.3 +_RESIDUAL_DIVISOR = math.hypot(1.0 - _RESIDUAL_BLEND, _RESIDUAL_BLEND) + + +def _rms_normalize(x: torch.Tensor, dim: int | tuple[int, ...]) -> torch.Tensor: + dims = (dim,) if isinstance(dim, int) else dim + element_count = math.prod(x.shape[axis] for axis in dims) + l2_norm = torch.linalg.vector_norm(x, dim=dims, keepdim=True, dtype=torch.float32) + rms = torch.add(_NORMALIZATION_EPSILON, l2_norm, alpha=element_count**-0.5) + return x / rms.to(x.dtype) + + +def _conv1d(in_channels: int, out_channels: int, kernel_size: int) -> nn.Conv1d: + return nn.Conv1d( + in_channels, out_channels, kernel_size, padding=kernel_size // 2, bias=False + ) + + +def _conv1d_with_gain( + conv: nn.Conv1d, x: torch.Tensor, gain: torch.Tensor | float +) -> torch.Tensor: + return F.conv1d( + x, + conv.weight * gain, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=conv.groups, + ) + + +class DiagonalGaussianDistribution: + def __init__(self, parameters: torch.Tensor, deterministic: bool = False) -> None: + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if deterministic: + self.var = self.std = torch.zeros_like(self.mean) + + def sample(self, generator: torch.Generator | None = None) -> torch.Tensor: + noise = torch.empty_like(self.mean).normal_(generator=generator) + return self.mean + self.std * noise + + def mode(self) -> torch.Tensor: + return self.mean + + +class ResnetBlock1D(nn.Module): + def __init__( + self, + *, + in_dim: int, + out_dim: int | None = None, + conv_shortcut: bool = False, + kernel_size: int = 3, + use_norm: bool = True, + ) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = in_dim if out_dim is None else out_dim + self.use_conv_shortcut = conv_shortcut + self.use_norm = use_norm + self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size) + self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size) + if self.in_dim != self.out_dim: + if conv_shortcut: + self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size) + else: + self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.use_norm: + x = _rms_normalize(x, dim=1) + hidden = self.conv1(F.silu(x) / _SILU_DIVISOR) + hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR) + if self.in_dim != self.out_dim: + shortcut = ( + self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut + ) + x = shortcut(x) + return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR + + +class AttnBlock1D(nn.Module): + def __init__(self, in_channels: int, num_heads: int = 1) -> None: + super().__init__() + self.in_channels = in_channels + self.num_heads = num_heads + self.qkv = _conv1d(in_channels, in_channels * 3, 1) + self.proj_out = _conv1d(in_channels, in_channels, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + qkv = self.qkv(x).reshape(x.shape[0], self.num_heads, -1, 3, x.shape[-1]) + query, key, value = _rms_normalize(qkv, dim=2).unbind(3) + query = rearrange(query, "b h c l -> b h l c") + key = rearrange(key, "b h c l -> b h l c") + value = rearrange(value, "b h c l -> b h l c") + hidden = F.scaled_dot_product_attention(query, key, value) + hidden = rearrange(hidden, "b h l c -> b (h c) l") + return torch.lerp(x, self.proj_out(hidden), _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR + + +class Upsample1D(nn.Module): + def __init__(self, in_channels: int, with_conv: bool) -> None: + super().__init__() + self.with_conv = with_conv + if with_conv: + self.conv = _conv1d(in_channels, in_channels, 3) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact") + return self.conv(x) if self.with_conv else x + + +class Downsample1D(nn.Module): + def __init__(self, in_channels: int, with_conv: bool) -> None: + super().__init__() + self.with_conv = with_conv + if with_conv: + self.conv1 = _conv1d(in_channels, in_channels, 1) + self.conv2 = _conv1d(in_channels, in_channels, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.with_conv: + x = self.conv1(x) + x = F.avg_pool1d(x, kernel_size=2, stride=2) + return self.conv2(x) if self.with_conv else x + + +class Encoder1D(nn.Module): + def __init__( + self, + *, + dim: int, + ch_mult: tuple[int, ...], + num_res_blocks: int, + attn_layers: list[int], + down_layers: list[int], + in_dim: int, + embed_dim: int, + resamp_with_conv: bool = True, + double_z: bool = True, + kernel_size: int = 3, + clip_act: float = 256.0, + ) -> None: + super().__init__() + self.dim = dim + self.num_layers = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.in_channels = in_dim + self.clip_act = clip_act + self.down_layers = down_layers + self.attn_layers = attn_layers + self.conv_in = _conv1d(in_dim, dim, kernel_size) + + in_ch_mult = (1,) + ch_mult + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for level in range(self.num_layers): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = dim * in_ch_mult[level] + block_out = dim * ch_mult[level] + for _ in range(num_res_blocks): + block.append( + ResnetBlock1D( + in_dim=block_in, + out_dim=block_out, + kernel_size=kernel_size, + use_norm=True, + ) + ) + block_in = block_out + if level in attn_layers: + attn.append(AttnBlock1D(block_in)) + down = nn.Module() + down.block = block + down.attn = attn + if level in down_layers: + down.downsample = Downsample1D(block_in, resamp_with_conv) + self.down.append(down) + + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock1D( + in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True + ) + self.mid.attn_1 = AttnBlock1D(block_in) + self.mid.block_2 = ResnetBlock1D( + in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True + ) + output_dim = 2 * embed_dim if double_z else embed_dim + self.conv_out = _conv1d(block_in, output_dim, kernel_size) + self.learnable_gain = nn.Parameter(torch.zeros([])) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + states = [self.conv_in(x)] + for level in range(self.num_layers): + for block_index in range(self.num_res_blocks): + hidden = self.down[level].block[block_index](states[-1]) + if len(self.down[level].attn) > 0: + hidden = self.down[level].attn[block_index](hidden) + states.append(hidden.clamp(-self.clip_act, self.clip_act)) + if level in self.down_layers: + states.append(self.down[level].downsample(states[-1])) + hidden = self.mid.block_1(states[-1]) + hidden = self.mid.attn_1(hidden) + hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act) + return _conv1d_with_gain( + self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1 + ) + + +class Decoder1D(nn.Module): + def __init__( + self, + *, + dim: int, + out_dim: int, + ch_mult: tuple[int, ...], + num_res_blocks: int, + attn_layers: list[int], + down_layers: list[int], + in_dim: int, + embed_dim: int, + kernel_size: int = 3, + resamp_with_conv: bool = True, + clip_act: float = 256.0, + ) -> None: + super().__init__() + self.ch = dim + self.num_layers = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.in_channels = in_dim + self.clip_act = clip_act + self.down_layers = [level + 1 for level in down_layers] + block_in = dim * ch_mult[-1] + self.conv_in = _conv1d(embed_dim, block_in, kernel_size) + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock1D( + in_dim=block_in, out_dim=block_in, use_norm=True + ) + self.mid.attn_1 = AttnBlock1D(block_in) + self.mid.block_2 = ResnetBlock1D( + in_dim=block_in, out_dim=block_in, use_norm=True + ) + + self.up = nn.ModuleList() + for level in reversed(range(self.num_layers)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = dim * ch_mult[level] + for _ in range(num_res_blocks + 1): + block.append( + ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True) + ) + block_in = block_out + if level in attn_layers: + attn.append(AttnBlock1D(block_in)) + up = nn.Module() + up.block = block + up.attn = attn + if level in self.down_layers: + up.upsample = Upsample1D(block_in, resamp_with_conv) + self.up.insert(0, up) + + self.conv_out = _conv1d(block_in, out_dim, kernel_size) + self.learnable_gain = nn.Parameter(torch.zeros([])) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + hidden = self.conv_in(z) + hidden = self.mid.block_1(hidden) + hidden = self.mid.attn_1(hidden) + hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act) + for level in reversed(range(self.num_layers)): + for block_index in range(self.num_res_blocks + 1): + hidden = self.up[level].block[block_index](hidden) + if len(self.up[level].attn) > 0: + hidden = self.up[level].attn[block_index](hidden) + hidden = hidden.clamp(-self.clip_act, self.clip_act) + if level in self.down_layers: + hidden = self.up[level].upsample(hidden) + return _conv1d_with_gain( + self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1 + ) + + +class MMAudioVAE(nn.Module): + """MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio.""" + + def __init__( + self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False + ) -> None: + super().__init__() + if isinstance(mode, dict): + config = mode + mode = config.get("mode", "44k") + need_encoder = config.get("need_encoder", need_encoder) + if mode == "16k": + data_dim, embed_dim, hidden_dim = 80, 20, 384 + data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D + elif mode == "44k": + data_dim, embed_dim, hidden_dim = 128, 40, 512 + data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D + else: + raise ValueError(f"Unknown MMAudio VAE mode: {mode}") + + self.mode = mode + self.embed_dim = embed_dim + self._weights_normalized = False + self.register_buffer( + "data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1) + ) + self.register_buffer( + "data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1) + ) + if need_encoder: + self.encoder = Encoder1D( + dim=hidden_dim, + ch_mult=(1, 2, 4), + num_res_blocks=2, + attn_layers=[3], + down_layers=[0], + in_dim=data_dim, + embed_dim=embed_dim, + ) + self.decoder = Decoder1D( + dim=hidden_dim, + ch_mult=(1, 2, 4), + num_res_blocks=2, + attn_layers=[3], + down_layers=[0], + in_dim=data_dim, + out_dim=data_dim, + embed_dim=embed_dim, + ) + + def encode( + self, mel: torch.Tensor, normalize_input: bool = True + ) -> DiagonalGaussianDistribution: + self._require_normalized_weights() + if not hasattr(self, "encoder"): + raise RuntimeError("This MMAudio VAE was loaded decoder-only") + if normalize_input: + mel = self.normalize(mel) + return DiagonalGaussianDistribution(self.encoder(mel)) + + def decode( + self, latent: torch.Tensor, unnormalize_output: bool = True + ) -> torch.Tensor: + self._require_normalized_weights() + mel = self.decoder(latent) + return self.unnormalize(mel) if unnormalize_output else mel + + def forward(self, latent: torch.Tensor) -> torch.Tensor: + return self.decode(latent) + + def normalize(self, mel: torch.Tensor) -> torch.Tensor: + return (mel - self.data_mean) / self.data_std + + def unnormalize(self, mel: torch.Tensor) -> torch.Tensor: + return mel * self.data_std + self.data_mean + + def _require_normalized_weights(self) -> None: + if not self._weights_normalized: + raise RuntimeError("call remove_weight_norm() before inference") + + @torch.no_grad() + def remove_weight_norm(self) -> MMAudioVAE: + # Idempotent: the diffusers reference calls this exactly once, in + # ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the + # freshly-constructed (random) Conv1d weights with a checkpoint's + # real, already-final values -- so a real checkpoint's own weights + # are never renormalized. Mirroring that timing here (see + # ``Kandinsky6AudioVAE.__init__``) means this method only ever runs + # against pre-load weights too; the guard below additionally makes a + # second, redundant call (e.g. the wrapper's own public + # ``remove_weight_norm()``) a true no-op instead of re-mutating an + # already-normalized (possibly checkpoint-loaded) weight a second + # time. + if self._weights_normalized: + return self + for name, module in self.named_modules(): + if isinstance(module, nn.Conv1d): + weight = _rms_normalize(module.weight.to(torch.float32), dim=(1, 2)) + weight = weight / math.sqrt(weight[0].numel()) + module.weight.copy_(weight.to(module.weight.dtype)) + logger.debug("Removed weight norm from %s", name) + self._weights_normalized = True + return self diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py new file mode 100644 index 0000000000..af2d2359a2 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/kandinsky6_sr_vae.py @@ -0,0 +1,656 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Causal Kandinsky 6 SR KVAE: spatial x16 and temporal x4. + +Encode/decode segments preserve causal-convolution caches; segmentation affects +numerics. Pixels normalize as x/128-1, encode returns an unscaled posterior mean, +and decode returns normalized pixels. encoder.* and decoder.* match the checkpoint.""" + +from __future__ import annotations + +import math +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.models.autoencoders.vae import DecoderOutput + +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEArchConfig, + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.runtime.distributed import ( + get_decode_parallel_rank, + get_decode_parallel_world_size, +) +from sglang.multimodal_gen.runtime.layers.parallel_conv import ( + SpatialParallelConv3d, + disable_spatial_parallel_decode, + gather_and_trim_height, + split_height_for_parallel_decode, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.models.vaes.common import ( + can_install_spatial_shard_parallel_decode, + should_run_spatial_shard_parallel_decode, +) +from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm + +# Pixel frames per encode segment after the first (which has one more: the causal first frame). +_SEGMENT_FRAMES = 16 +# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles +# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9 +# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv +# workspace to a fraction of the activation. +_MAX_CONV_NUMEL = 2 * 10**9 + + +class _SegmentCache: + """Causal state carried from one temporal segment to the next during a single encode / decode call.""" + + def __init__(self) -> None: + self.first = True + self.padding: dict[nn.Module, torch.Tensor] = {} + + +def _silu(x: torch.Tensor) -> torch.Tensor: + # The output norms use this form and the resblocks use F.silu; the two are not bitwise + # equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is + # deliberate, not a simplification opportunity. + return x * torch.sigmoid(x) + + +class _K6RMSNorm(WanRMS_norm): + """``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the + Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for + fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably + drifts from the reference over many resnet blocks.""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + normed = F.normalize(x.float(), dim=1).to(x.dtype) + return normed * self.scale * self.gamma + self.bias + + +class _ChunkedConv3d(nn.Conv3d): + """``nn.Conv3d`` that convolves inputs larger than ``_MAX_CONV_NUMEL`` in overlapping + temporal chunks (plain ``F.conv3d`` silently corrupts most elements once the input exceeds + ~2**31 elements: it appears to do 32-bit index arithmetic internally).""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + num_parts = math.ceil(x.numel() / _MAX_CONV_NUMEL) + if num_parts <= 1: + return super().forward(x) + k, frames = self.kernel_size[0], x.size(2) + if self.padding[0] != 0: + raise ValueError( + f"temporal chunking needs an unpadded time axis, got kernel {self.kernel_size}, " + f"padding {self.padding}" + ) + if self.stride[0] != 1: + # A strided conv can't reuse the stride-1 path's "prefix each chunk with the + # previous chunk's tail" trick below (a strided conv's output doesn't align with + # input chunk boundaries), so slide one kernel-width window at a time instead, only + # raising if even a single window would overflow. + stride = self.stride[0] + window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4] + if window_numel >= _MAX_CONV_NUMEL: + raise ValueError( + f"frames are too big for Conv3d even one stride-{stride} window at a time " + f"({window_numel} elements per window)" + ) + # Bare super() doesn't resolve inside a list comprehension's own implicit scope, + # hence the loop. + window_outputs = [] + for i in range(0, frames - k + 1, stride): + window_outputs.append(super().forward(x[:, :, i : i + k])) + return torch.cat(window_outputs, dim=2) + step = math.ceil(frames / num_parts) + last = frames - step * (math.ceil(frames / step) - 1) + if k > 1 and min(step, last) < k: + # A chunk would be shorter than the kernel: convolve one output frame at a time instead. + windows = [(i, i + k) for i in range(frames - k + 1)] + else: + # Each input chunk [start, end) is prefixed with the previous chunk's last k - 1 frames. + windows = [ + (max(start - k + 1, 0), min(start + step, frames)) + for start in range(0, frames, step) + ] + out: torch.Tensor | None = None + for lo, hi in windows: + y = super().forward(x[:, :, lo:hi]) + if out is None: + out = y.new_empty((*y.shape[:2], frames - k + 1, *y.shape[3:])) + out[:, :, lo : lo + y.size(2)] = y + assert out is not None + return out + + +class _SpatialChunkedConv3d(SpatialParallelConv3d, _ChunkedConv3d): + # exchange halos once before temporal chunking; uneven height shards may need + # different chunk counts, so collectives must stay outside that loop + @classmethod + def from_conv(cls, conv: _ChunkedConv3d, height_padding=None): + with torch.device("meta"): + parallel = cls( + conv.in_channels, + conv.out_channels, + conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=conv.groups, + bias=conv.bias is not None, + height_padding=height_padding, + ) + parallel.weight, parallel.bias = conv.weight, conv.bias + return parallel + + def _direct_forward(self, x): + x = F.pad(x, (0, 0, self.height_pad_top, self.height_pad_bottom, 0, 0)) + return _ChunkedConv3d.forward(self, x) + + +class _CausalConv3d(nn.Module): + """Conv3d that is causal in time: the first segment is left-padded with copies of its + first frame, later segments with the tail of the previous one; height / width are zero + padded.""" + + def __init__( + self, + chan_in: int, + chan_out: int, + kernel_size: int | tuple[int, int, int], + stride: tuple[int, int, int] = (1, 1, 1), + ) -> None: + super().__init__() + if isinstance(kernel_size, int): + kernel_size = (kernel_size,) * 3 + time_kernel, height_kernel, width_kernel = kernel_size + self.height_pad = height_kernel // 2 + self.width_pad = width_kernel // 2 + self.time_pad = time_kernel - 1 + self.time_kernel = time_kernel + self.time_stride = stride[0] + self.conv = _ChunkedConv3d(chan_in, chan_out, kernel_size, stride=stride) + + def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + s, k = self.time_stride, self.time_kernel + batch, _, frames, height, width = x.shape + x = F.pad( + x, (self.width_pad, self.width_pad, self.height_pad, self.height_pad, 0, 0) + ) + if cache.first: + first_frame = x[:, :, :1] + padding = first_frame.expand(-1, -1, self.time_pad, -1, -1) + else: + padding = cache.padding[self] + + # conv(cat([padding, x])) without materializing the concatenation of the whole segment: + # only the outputs whose window overlaps ``padding`` are computed on a small + # concatenated head. + out_frames = frames if s == 1 else (frames + 1) // 2 + output = x.new_empty((batch, self.conv.out_channels, out_frames, height, width)) + head_out = math.ceil(padding.size(2) / s) + head_in = head_out * s - padding.size(2) + if head_out > 0: + output[:, :, :head_out] = self.conv( + torch.cat([padding, x[:, :, : head_in + k - s]], dim=2) + ) + if head_out < out_frames: + output[:, :, head_out:] = self.conv(x[:, :, head_in:]) + + # Keep exactly the input frames the next segment's first output window needs. + pad_offset = head_in + s * math.trunc((frames - head_in - k) / s) + s + if pad_offset < 0: # segment shorter than the kernel + cache.padding[self] = torch.cat([padding[:, :, pad_offset:], x], dim=2) + else: + cache.padding[self] = x[:, :, pad_offset:].clone() + return output + + +class _ResnetBlock3D(nn.Module): + """norm -> SiLU -> causal conv, twice, plus a 1x1 shortcut when the width changes. Decoder + blocks use the latent-conditioned ``_SpatialNorm3D`` (``zq_channels`` set).""" + + def __init__( + self, in_channels: int, out_channels: int, zq_channels: int | None = None + ) -> None: + super().__init__() + if zq_channels is None: + self.norm1: nn.Module = _K6RMSNorm(in_channels, images=False) + self.norm2: nn.Module = _K6RMSNorm(out_channels, images=False) + else: + self.norm1 = _SpatialNorm3D(in_channels, zq_channels) + self.norm2 = _SpatialNorm3D(out_channels, zq_channels) + self.conv1 = _CausalConv3d(in_channels, out_channels, kernel_size=3) + self.conv2 = _CausalConv3d(out_channels, out_channels, kernel_size=3) + if in_channels != out_channels: + self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1) + + def _norm( + self, + norm: nn.Module, + h: torch.Tensor, + zq: torch.Tensor | None, + cache: _SegmentCache, + ) -> torch.Tensor: + return norm(h) if zq is None else norm(h, zq, cache) + + def forward( + self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None + ) -> torch.Tensor: + h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True) + h = self.conv1(h, cache) + h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True) + h = self.conv2(h, cache) + if hasattr(self, "nin_shortcut"): + x = self.nin_shortcut(x) + return x + h + + +def _chunked_interpolate_nearest( + x: torch.Tensor, size: tuple[int, int, int], channels: int = 32 +) -> torch.Tensor: + """Nearest interpolation in channel chunks, bounding memory and int32 indexing.""" + if x.shape[1] <= channels: + return F.interpolate(x, size=size, mode="nearest") + return torch.cat( + [ + F.interpolate(chunk, size=size, mode="nearest") + for chunk in torch.split(x, channels, dim=1) + ], + dim=1, + ) + + +class _SpatialNorm3D(nn.Module): + """RMS norm modulated by the latent: ``norm(f) * conv_y(zq) + conv_b(zq)`` with ``zq`` + nearest-resized to ``f``.""" + + def __init__(self, f_channels: int, zq_channels: int) -> None: + super().__init__() + self.norm_layer = _K6RMSNorm(f_channels, images=False) + self.conv_y = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1) + self.conv_b = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1) + + def forward( + self, f: torch.Tensor, zq: torch.Tensor, cache: _SegmentCache + ) -> torch.Tensor: + frames, height, width = f.shape[-3:] + if cache.first: + # The causal first latent frame maps to exactly one pixel frame; the rest cover the + # remaining frames. + zq_first = _chunked_interpolate_nearest(zq[:, :, :1], (1, height, width)) + if zq.size(2) > 1: + zq_rest = _chunked_interpolate_nearest( + zq[:, :, 1:], (frames - 1, height, width) + ) + zq = torch.cat([zq_first, zq_rest], dim=2) + else: + zq = zq_first + else: + zq = _chunked_interpolate_nearest(zq, (frames, height, width)) + norm_f = self.norm_layer(f) + norm_f.mul_(self.conv_y(zq)) + norm_f.add_(self.conv_b(zq)) + return norm_f + + +class _PXSDownsample(nn.Module): + """x2 spatial (strided conv + channel-averaged pixel-unshuffle) and optional x2 causal + temporal (two causal convs + average pooling) downsample; doubles the channels.""" + + def __init__(self, in_channels: int, compress_time: bool) -> None: + super().__init__() + out_channels = 2 * in_channels + self.spatial_conv = _ChunkedConv3d( + in_channels, + out_channels, + kernel_size=(1, 3, 3), + stride=(1, 2, 2), + padding=(0, 1, 1), + ) + self.compress_time = compress_time + if compress_time: + self.temporal_conv = nn.Sequential( + _CausalConv3d(out_channels, out_channels, kernel_size=(2, 1, 1)), + _CausalConv3d( + out_channels, out_channels, kernel_size=(2, 1, 1), stride=(2, 1, 1) + ), + ) + self.linear = _ChunkedConv3d(out_channels, out_channels, kernel_size=1) + + def _spatial(self, x: torch.Tensor) -> torch.Tensor: + b, c, t, h, w = x.shape + pxs = F.pixel_unshuffle(x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w), 2) + pxs = pxs.view(b * t, 2 * c, 2, h // 2, w // 2).mean(dim=2) + pxs = pxs.reshape(b, t, 2 * c, h // 2, w // 2).permute(0, 2, 1, 3, 4) + return self.spatial_conv(x) + pxs + + def _temporal(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + b, c, t, h, w = x.shape + pooled = x.permute(0, 3, 4, 1, 2).reshape(b * h * w, c, t) + if cache.first: # the causal first frame is kept as is + first, rest = pooled[..., :1], pooled[..., 1:] + pooled = ( + torch.cat([first, F.avg_pool1d(rest, kernel_size=2, stride=2)], dim=-1) + if t > 1 + else first + ) + else: + pooled = F.avg_pool1d(pooled, kernel_size=2, stride=2) + pooled = pooled.reshape(b, h, w, c, -1).permute(0, 3, 4, 1, 2) + conv = self.temporal_conv[1](self.temporal_conv[0](x, cache), cache) + return conv + pooled + + def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + out = self._spatial(x) + if self.compress_time: + out = self._temporal(out, cache) + return self.linear(out) + + +class _PXSUpsample(nn.Module): + """Optional x2 causal temporal (frame repeat + causal conv residual) then x2 spatial + (nearest + conv residual) upsample.""" + + def __init__(self, in_channels: int, compress_time: bool) -> None: + super().__init__() + self.spatial_conv = _ChunkedConv3d( + in_channels, in_channels, kernel_size=(1, 3, 3), padding=(0, 1, 1) + ) + self.compress_time = compress_time + if compress_time: + self.temporal_conv = _CausalConv3d( + in_channels, in_channels, kernel_size=(3, 1, 1) + ) + self.linear = _ChunkedConv3d(in_channels, in_channels, kernel_size=1) + + def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + if self.compress_time: + x = x.repeat_interleave(2, dim=2) + if cache.first: # the causal first frame is not duplicated + x = x[:, :, 1:] + x = self.temporal_conv(x, cache) + x + frames, height, width = x.shape[-3:] + # Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow + # this guards against (the old vendored code did not chunk this specific call). + x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2)) + x.add_(self.spatial_conv(x)) + return self.linear(x) + + +def _level_module(**children: nn.Module) -> nn.Module: + module = nn.Module() + for name, child in children.items(): + setattr(module, name, child) + return module + + +class _Encoder3D(nn.Module): + def __init__( + self, + *, + ch: int, + ch_mult: list[float], + num_res_blocks: int, + in_channels: int, + z_channels: int, + temporal_compress_times: int, + temporal_compress_start_level: int, + ) -> None: + super().__init__() + self.num_res_blocks = num_res_blocks + time_levels = range( + temporal_compress_start_level, + temporal_compress_start_level + int(math.log2(temporal_compress_times)), + ) + self.conv_in = _CausalConv3d(in_channels, round(ch * ch_mult[0]), kernel_size=3) + self.down = nn.ModuleList() + block_in = round(ch * ch_mult[0]) + for level, mult in enumerate(ch_mult): + block_out = round(ch * mult) + blocks = nn.ModuleList() + for _ in range(num_res_blocks): + blocks.append(_ResnetBlock3D(block_in, block_out)) + block_in = block_out + down = _level_module(block=blocks) + if level != len(ch_mult) - 1: + down.downsample = _PXSDownsample( + block_in, compress_time=level in time_levels + ) + block_in *= 2 + self.down.append(down) + self.mid = _level_module( + block_1=_ResnetBlock3D(block_in, block_in), + block_2=_ResnetBlock3D(block_in, block_in), + ) + self.norm_out = _K6RMSNorm(block_in, images=False) + self.conv_out = _CausalConv3d(block_in, 2 * z_channels, kernel_size=3) + + def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + h = self.conv_in(x, cache) + for down in self.down: + for block in down.block: + h = block(h, cache) + if hasattr(down, "downsample"): + h = down.downsample(h, cache) + h = self.mid.block_2(self.mid.block_1(h, cache), cache) + return self.conv_out(_silu(self.norm_out(h)), cache) + + +class _Decoder3D(nn.Module): + def __init__( + self, + *, + ch: int, + ch_mult: list[float], + num_res_blocks: int, + out_ch: int, + z_channels: int, + temporal_compress_times: int, + temporal_compress_start_level: int, + ) -> None: + super().__init__() + num_levels = len(ch_mult) + # Mirror of the encoder's temporally compressing levels. + time_levels = range( + num_levels + - temporal_compress_start_level + - int(math.log2(temporal_compress_times)), + num_levels - temporal_compress_start_level, + ) + block_in = round(ch * ch_mult[-1]) + self.conv_in = _CausalConv3d(z_channels, block_in, kernel_size=3) + self.mid = _level_module( + block_1=_ResnetBlock3D(block_in, block_in, z_channels), + block_2=_ResnetBlock3D(block_in, block_in, z_channels), + ) + up_levels: list[nn.Module] = [] + for level in reversed(range(num_levels)): + block_out = round(ch * ch_mult[level]) + blocks = nn.ModuleList() + for _ in range(num_res_blocks + 1): + blocks.append(_ResnetBlock3D(block_in, block_out, z_channels)) + block_in = block_out + up = _level_module(block=blocks) + if level != 0: + up.upsample = _PXSUpsample(block_in, compress_time=level in time_levels) + up_levels.insert(0, up) + self.up = nn.ModuleList(up_levels) + self.norm_out = _SpatialNorm3D(block_in, z_channels) + self.conv_out = _CausalConv3d(block_in, out_ch, kernel_size=3) + + def forward(self, z: torch.Tensor, cache: _SegmentCache) -> torch.Tensor: + h = self.conv_in(z, cache) + h = self.mid.block_2(self.mid.block_1(h, cache, z), cache, z) + for up in reversed(self.up): + for block in up.block: + h = block(h, cache, z) + if hasattr(up, "upsample"): + h = up.upsample(h, cache) + return self.conv_out(_silu(self.norm_out(h, z, cache)), cache) + + +# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything +# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``, +# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored +# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params. +_ARCH_KEYS = ( + "ch", + "ch_mult", + "num_res_blocks", + "z_channels", + "temporal_compress_times", +) + + +def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]: + # norm_type is the one legacy knob this port still checks: unlike padding_mode / + # downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer + # branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride / + # double-z behavior those knobs used to select), a group_norm checkpoint would silently + # get rms_norm layers instead of a clear error, which is the one mismatch worth failing + # loudly on here. + norm_type = conf.get("norm_type", "rms_norm") + if norm_type != "rms_norm": + raise ValueError( + f"Kandinsky6SRVAE supports only {which}_config['norm_type'] == 'rms_norm' " + f"(group-norm KVAEs are not implemented by this port), got {norm_type!r}" + ) + missing = [key for key in (*_ARCH_KEYS, extra) if key not in conf] + if missing: + raise ValueError(f"Kandinsky6SRVAE {which}_config is missing {missing}") + kwargs = {key: conf[key] for key in (*_ARCH_KEYS, extra)} + kwargs["temporal_compress_start_level"] = conf.get( + "temporal_compress_start_level", 0 + ) + return kwargs + + +class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin): + """Segment-cached causal video KVAE used by the SR pipeline.""" + + layerwise_offload_dit_group_enabled = False + + def __init__(self, config: Kandinsky6SRVAEConfig) -> None: + super().__init__() + self.config = config + arch = config.arch_config + assert isinstance(arch, Kandinsky6SRVAEArchConfig) + if not arch.encoder_config or not arch.decoder_config: + raise ValueError( + "Kandinsky6SRVAE needs `encoder_config` and `decoder_config` (KVAE " + "architecture) in the component config.json." + ) + enc = _arch_kwargs(dict(arch.encoder_config), "in_channels", "encoder") + dec = _arch_kwargs(dict(arch.decoder_config), "out_ch", "decoder") + self.temporal_compression = enc["temporal_compress_times"] + self.encoder = _Encoder3D(**enc) + self.decoder = _Decoder3D(**dec) + self._spatial_parallel_decode_enabled = ( + can_install_spatial_shard_parallel_decode(config) + ) + if self._spatial_parallel_decode_enabled: + for module in self.decoder.modules(): + if isinstance(module, _CausalConv3d) and module.height_pad: + module.conv = _SpatialChunkedConv3d.from_conv( + module.conv, (module.height_pad, module.height_pad) + ) + module.height_pad = 0 + elif isinstance(module, _PXSUpsample): + module.spatial_conv = _SpatialChunkedConv3d.from_conv( + module.spatial_conv + ) + # encode/decode invoke each block, not the surrounding level container + self.layer_names = [ + f"{path}.{index}.block" + for path, levels in ( + ("encoder.down", self.encoder.down), + ("decoder.up", self.decoder.up), + ) + for index in range(len(levels)) + ] + + @property + def scaling_factor(self) -> float: + return float(self.config.arch_config.scaling_factor) + + @property + def spatial_factor(self) -> int: + return int(self.config.arch_config.spatial_factor) + + @property + def temporal_factor(self) -> int: + return int(self.config.arch_config.temporal_factor) + + @staticmethod + def normalize_data(data: torch.Tensor) -> torch.Tensor: + return data / 128 - 1.0 + + @staticmethod + def denormalize_data(data: torch.Tensor) -> torch.Tensor: + return (data + 1) * 128 + + def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]: + """``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``.""" + split_list = [_SEGMENT_FRAMES + 1] + remaining = x.size(2) - split_list[0] + while remaining > 0: + split_list.append(_SEGMENT_FRAMES) + remaining -= _SEGMENT_FRAMES + split_list[-1] += remaining + cache = _SegmentCache() + latents = [] + for segment in torch.split(x, split_list, dim=2): + moments = self.encoder(segment, cache) + cache.first = False + latents.append( + moments.chunk(2, dim=1)[0] + ) # deterministic posterior: the mean + return torch.cat(latents, dim=2), split_list + + def decode(self, z: torch.Tensor) -> DecoderOutput: + """``[B, C, T', h, w]`` raw latent -> ``.sample`` ``[B, 3, T, H, W]`` in the normalized pixel range.""" + if self._spatial_parallel_decode_enabled: + world_size = get_decode_parallel_world_size() + if ( + should_run_spatial_shard_parallel_decode(self.config, z) + and z.shape[-2] >= world_size + ): + z, height = split_height_for_parallel_decode( + z, + expected_height=z.shape[-2] * self.spatial_factor, + world_size=world_size, + rank=get_decode_parallel_rank(), + ) + return DecoderOutput( + sample=gather_and_trim_height(self._decode(z), height) + ) + with disable_spatial_parallel_decode(): + return DecoderOutput(sample=self._decode(z)) + return DecoderOutput(sample=self._decode(z)) + + def _decode(self, z: torch.Tensor) -> torch.Tensor: + segment = _SEGMENT_FRAMES // self.temporal_compression + num_frames = z.size(2) + if num_frames == 1: + split_list = [1] + else: + split_list = [segment] * ((num_frames - 1) // segment) + if (num_frames - 1) % segment: + split_list.append((num_frames - 1) % segment) + split_list[0] += 1 + cache = _SegmentCache() + samples = [] + for chunk in torch.split(z, split_list, dim=2): + samples.append(self.decoder(chunk, cache)) + cache.first = False + return torch.cat(samples, dim=2) + + +EntryClass = Kandinsky6SRVAE + +__all__ = ["Kandinsky6SRVAE"] diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py index 77ccb3870a..14a720deb0 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_audio_vae/bigvgan.py @@ -3,13 +3,19 @@ # Licensed under the MIT license. # Adapted from https://github.com/jik876/hifi-gan under the MIT license. +# +# The alias-free resampling primitives (LowPassFilter1d/UpSample1d/ +# DownSample1d/Activation1d) are architecture-agnostic DSP building blocks +# shared with the sibling ``kandinsky6_audio_vae/bigvgan.py`` -- both import +# ``Activation1d`` from ``runtime/models/vaes/alias_free.py`` (sibling of +# this package, alongside ``common.py``) instead of each keeping its own copy. import torch import torch.nn as nn from torch.nn import Conv1d, ConvTranspose1d, Parameter from torch.nn.utils.parametrizations import weight_norm -from .alias_free import Activation1d +from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d def get_padding(kernel_size, dilation=1): diff --git a/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_pipeline.py new file mode 100644 index 0000000000..587782debc --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_pipeline.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Joint video/audio generation with optional image conditioning. + +One stage chain handles both T2VA and IT2VA; the image stage is a no-op for T2VA.""" + +from __future__ import annotations + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6 import ( + Kandinsky6TI2VASamplingParams, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline +from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import ( + Kandinsky6AudioDecodingStage, + Kandinsky6DecodingStage, + Kandinsky6DenoisingStage, + Kandinsky6ImageEncodingStage, + Kandinsky6LatentPreparationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import ( + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class Kandinsky6TI2VAPipeline(LoRAPipeline, ComposedPipelineBase): + pipeline_name = "Kandinsky6TI2VAPipeline" + is_video_pipeline = True + pipeline_config_cls = Kandinsky6TI2VAPipelineConfig + sampling_params_cls = Kandinsky6TI2VASamplingParams + + # Kandinsky6AudioVAE bundles the mel-VAE decoder *and* the BigVGAN-v2 + # vocoder in one checkpoint component -- there is no separate "vocoder" + # module, unlike LTX-2's separately-loaded audio_vae/vocoder pair. + _required_config_modules = [ + "scheduler", + "text_encoder", + "text_encoder_2", + "tokenizer", + "tokenizer_2", + "transformer", + "vae", + "audio_vae", + ] + + def validate_disagg_role(self, role: RoleType) -> None: + # coupled denoising and reference-frame state are not yet disaggregation-aware + if role != RoleType.MONOLITHIC: + raise ValueError( + "Kandinsky6TI2VAPipeline only supports monolithic deployment; " + f"disaggregation role {role.value!r} is not supported" + ) + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + self.add_stage( + stage_name="input_validation_stage", stage=InputValidationStage() + ) + + self.add_stage( + stage_name="text_encoding_stage", + stage=TextEncodingStage( + text_encoders=[ + self.get_module("text_encoder"), + self.get_module("text_encoder_2"), + ], + tokenizers=[ + self.get_module("tokenizer"), + self.get_module("tokenizer_2"), + ], + ), + ) + + self.add_stage( + stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")), + ) + + self.add_stage( + stage_name="latent_preparation_stage", + stage=Kandinsky6LatentPreparationStage(), + ) + + # No-op unless the request supplies a conditioning image; see + # Kandinsky6ImageEncodingStage's docstring. + self.add_stage( + stage_name="image_encoding_stage", + stage=Kandinsky6ImageEncodingStage(vae=self.get_module("vae")), + ) + + self.add_stage( + stage_name="denoising_stage", + stage=Kandinsky6DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + pipeline=self, + ), + ) + + # Audio decodes before video, matching the diffusers reference and + # FastVideo's own stage order -- do not reorder. + self.add_stage( + stage_name="audio_decoding_stage", + stage=Kandinsky6AudioDecodingStage(audio_vae=self.get_module("audio_vae")), + ) + + self.add_stage( + stage_name="decoding_stage", + stage=Kandinsky6DecodingStage(vae=self.get_module("vae"), pipeline=self), + ) + + +EntryClass = [Kandinsky6TI2VAPipeline] diff --git a/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py new file mode 100644 index 0000000000..57148dee39 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines/kandinsky6_sr_pipeline.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 video SR: prepare, encode/upscale, denoise, decode and blend. + +Work is grouped by component to avoid weight transfers per tile. The bundle's +scheduler drives denoising through the shared DenoisingStage lifecycle. +num_inference_steps counts DiT calls; pi-Flow uses its checkpoint-defined nfe.""" + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( + Kandinsky6SRDecodeStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( + Kandinsky6SRDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import ( + Kandinsky6SREncodeStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( + Kandinsky6SRInputStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( + Kandinsky6SRLatentPrepStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import ( + Kandinsky6SROutputStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class Kandinsky6SRPipeline(ComposedPipelineBase): + pipeline_name = "Kandinsky6SRPipeline" + is_video_pipeline = True + pipeline_config_cls = Kandinsky6SRPipelineConfig + sampling_params_cls = Kandinsky6SRSamplingParams + + _required_config_modules = ["transformer", "vae", "scheduler", "latent_upscaler"] + # Repos without a latent-upscaler bank run the pixel path only. + _optional_config_modules = ("latent_upscaler",) + + def validate_disagg_role(self, role: RoleType) -> None: + # The tiled loop keeps tile latents / decoded tiles in one process. + if role != RoleType.MONOLITHIC: + raise ValueError( + "Kandinsky6SRPipeline only supports monolithic deployment; " + f"disaggregation role {role.value!r} is not supported" + ) + + def load_modules(self, server_args: ServerArgs, loaded_modules=None): + model_index = self._load_config() + if "dit" in model_index and "transformer" not in model_index: + raise ValueError( + f"{self.model_path} is a k6_video SR bundle (it has a 'dit' component), " + "not an official Diffusers Kandinsky6SRPipeline repo. Use " + "kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers or " + "kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers" + ) + for name in self._optional_config_modules: + if model_index.get(name) is None and name in self._required_config_modules: + self._required_config_modules.remove(name) + model_index.pop(name, None) + return super().load_modules(server_args, loaded_modules) + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + latent_upscaler = self.get_module("latent_upscaler") + vae = self.get_module("vae") + transformer = self.get_module("transformer") + scheduler = self.get_module("scheduler") + self.add_stage(stage_name="input_stage", stage=Kandinsky6SRInputStage()) + self.add_stage( + stage_name="encode_stage", + stage=Kandinsky6SREncodeStage(vae=vae, latent_upscaler=latent_upscaler), + ) + self.add_stage( + stage_name="latent_prep_stage", + stage=Kandinsky6SRLatentPrepStage( + vae=vae, + transformer=transformer, + latent_upscaler=latent_upscaler, + scheduler=scheduler, + ), + ) + self.add_stage( + stage_name="denoising_stage", + stage=Kandinsky6SRDenoisingStage( + transformer=transformer, scheduler=scheduler, pipeline=self + ), + ) + self.add_stage( + stage_name="decode_stage", stage=Kandinsky6SRDecodeStage(vae=vae) + ) + self.add_stage(stage_name="output_stage", stage=Kandinsky6SROutputStage()) + + +EntryClass = [Kandinsky6SRPipeline] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py index 454a4d39be..4ebeb3f0cd 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py @@ -488,6 +488,12 @@ class OutputBatch: raw_frame_metadata: dict[str, Any] | None = None audio: torch.Tensor | None = None audio_sample_rate: int | None = None + # The effective fps the worker actually produced ``output`` at (e.g. a model whose input + # stage resamples / resolves the request's nominal fps to a source-derived one, such as + # Kandinsky6 SR). ``None`` means the worker did not resolve its own fps; callers that save + # ``output`` client-side (return_file_paths_only=False) then fall back to the originating + # request's own fps field, as before this field existed. + fps: int | None = None action_pred: torch.Tensor | None = None action_mode: str | None = None action_domain_id: int | None = None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index bd6a0886ce..5a83786c42 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -1035,8 +1035,16 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): self._cache_dit_request_overrides = resolve_cache_dit_request_overrides( batch.sampling_params.cache_dit_params ) + has_separate_cfg = ( + requested + and batch.do_classifier_free_guidance + and not self.server_args.enable_cfg_parallel + ) desired_key = ( - cache_dit_overrides_key(self._cache_dit_request_overrides) + ( + cache_dit_overrides_key(self._cache_dit_request_overrides), + has_separate_cfg, + ) if requested else None ) @@ -1063,11 +1071,11 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): steps_computation_policy=scm_policy, ) else: - scm_preset = None if scm_preset == "none" else scm_preset refresh_context_on_transformer( self.transformer, primary_num_steps, - scm_preset=scm_preset, + steps_computation_mask=steps_computation_mask, + steps_computation_policy=scm_policy, ) return @@ -1156,7 +1164,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): model_name="transformer", sp_group=sp_group, tp_group=tp_group, - has_separate_cfg=batch.do_classifier_free_guidance, + has_separate_cfg=has_separate_cfg, ) logger.info( "cache-dit enabled on transformer (steps=%d, Fn=%d, Bn=%d, rdt=%.3f)", diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/__init__.py new file mode 100644 index 0000000000..a5ab771ffd --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Kandinsky6-specific pipeline stages.""" + +from .decoding import Kandinsky6AudioDecodingStage, Kandinsky6DecodingStage +from .denoising import Kandinsky6DenoisingStage +from .image_encoding import Kandinsky6ImageEncodingStage +from .latent_preparation import Kandinsky6LatentPreparationStage + +__all__ = [ + "Kandinsky6AudioDecodingStage", + "Kandinsky6DecodingStage", + "Kandinsky6DenoisingStage", + "Kandinsky6ImageEncodingStage", + "Kandinsky6LatentPreparationStage", +] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py new file mode 100644 index 0000000000..eddc22a675 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/decoding.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Decode audio before video, preserving the reference component order.""" + +from __future__ import annotations + +import torch + +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class Kandinsky6DecodingStage(DecodingStage): + """Convert video to channel-first for shared VAE decode; attach the decoded audio.""" + + def __init__(self, vae, pipeline=None) -> None: + super().__init__(vae=vae, pipeline=pipeline) + + def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: + if batch.latents is None: + raise ValueError("latents must be available before Kandinsky6 decoding.") + batch.latents = batch.latents.permute(0, 4, 1, 2, 3).contiguous() + output_batch = super().forward(batch, server_args) + output_batch.audio = batch.audio + output_batch.audio_sample_rate = batch.audio_sample_rate + return output_batch + + +class Kandinsky6AudioDecodingStage(PipelineStage): + """Decode audio through the bundled mel VAE/vocoder into batch.audio.""" + + def __init__(self, audio_vae) -> None: + super().__init__() + self.audio_vae = audio_vae + + @property + def role_affinity(self) -> RoleType: + return RoleType.DECODER + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + stage_name = self._component_stage_name(stage_name) + return [ComponentUse(stage_name, "audio_vae")] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + if batch.audio_latents is None: + return batch + + device = get_local_torch_device() + with self.use_declared_component( + component_name="audio_vae", module=self.audio_vae + ) as audio_vae: + assert audio_vae is not None + self.audio_vae = audio_vae + + # Cast to the module's own loaded weight dtype directly -- no + # autocast / precision-config knob, matching the existing + # MMAudio-lineage audio decoding stages in this codebase. + decoder_dtype = next(audio_vae.parameters()).dtype + + # apply checkpoint latent denormalization before the VAE's mel denormalization + scaling_factor = audio_vae.scaling_factor + mean_value = audio_vae.mean_value + latents = (batch.audio_latents / scaling_factor) + mean_value + + # [B, A, D] -> [B, D, A] to match the audio VAE's 1D-conv + # (channel, length) convention. + latents = latents.to(device=device, dtype=decoder_dtype).transpose(1, 2) + waveform = audio_vae.wrapped_decode(latents) # [B, 1, samples] + + # preserve [B, samples] so the save path selects the audio track for each output + batch.audio = waveform[:, 0].clamp(-1.0, 1.0).float().cpu() + batch.audio_sample_rate = int(server_args.pipeline_config.audio_sample_rate) + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("audio_latents", batch.audio_latents, V.none_or_tensor) + return result diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py new file mode 100644 index 0000000000..ae61db06cf --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/denoising.py @@ -0,0 +1,442 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Joint video/audio denoising using the shared stage lifecycle, BCG and CFG. + +Flow-Euler advances the video scheduler once per step; audio uses the same +sigma delta without advancing that scheduler again. pi-Flow uses independent +video/audio schedulers and retains fp32 state between bf16 DiT calls. + +Adapted from FastVideo's Kandinsky6DenoisingStage.""" + +from __future__ import annotations + +import dataclasses +import math +from copy import deepcopy +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import ( + run_cfg_parallel, + run_two_branch_cfg_parallel, +) +from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_classifier_free_guidance_world_size, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import ( + TAIL_COND_ACTIVE_EXTRA_KEY, + VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE + + +class Kandinsky6DenoisingStage(DenoisingStage): + """Run the Kandinsky6 joint video+audio denoising loop.""" + + def __init__(self, transformer, scheduler, pipeline=None) -> None: + super().__init__( + transformer=transformer, scheduler=scheduler, pipeline=pipeline + ) + + def _owns_compile_warmup_lifecycle(self) -> bool: + # forward() overrides DenoisingStage.forward (the joint video+audio + # loop cannot reuse the parent's single-tensor _denoise()), but it + # still wraps the loop in `_offload_for_torch_compile_warmup` itself + # below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's + # other joint-modality denoising stage. + return True + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + stage_name = self._component_stage_name(stage_name) + return [ + ComponentUse( + stage_name, + "transformer", + phase="transformer", + preferred_ready_after_request=True, + memory_intensive=True, + ) + ] + + @staticmethod + def _text_rope_pos(mask: torch.Tensor, device: torch.device) -> torch.Tensor: + seq_len = int(mask.sum(1).max().item()) + return torch.arange(seq_len, device=device) + + def _call_transformer(self, transformer, **call_kwargs) -> Any: + """Dispatch through BCG when enabled, otherwise call the transformer directly.""" + runner = self._maybe_get_bcg_runner(transformer) + if runner is not None: + return self._bcg_run(runner, call_kwargs, transformer) + return transformer(**call_kwargs) + + def _build_cfg_policy( + self, + batch: Req, + server_args: ServerArgs, + *, + prompt_embeds: torch.Tensor, + pooled: torch.Tensor, + text_rope_pos: torch.Tensor, + ) -> CFGPolicy: + """Reuse the pipeline CFG policy with joint-modality conditioning kwargs.""" + branches = [ + CFGBranch( + "conditional", + True, + { + "encoder_hidden_states": prompt_embeds, + "pooled_projections": pooled, + "text_rope_pos": text_rope_pos, + }, + ) + ] + if batch.do_classifier_free_guidance and batch.negative_prompt_embeds: + neg_prompt_embeds = server_args.pipeline_config.get_neg_prompt_embeds( + batch + ).to(device=prompt_embeds.device, dtype=prompt_embeds.dtype) + if not batch.neg_pooled_embeds: + raise ValueError( + "Kandinsky6 requires CLIP negative pooled projections for CFG." + ) + neg_pooled = batch.neg_pooled_embeds[0].to( + device=prompt_embeds.device, dtype=prompt_embeds.dtype + ) + if not batch.negative_attention_mask: + raise ValueError( + "Kandinsky6 requires Qwen (Reason1) negative attention masks for CFG." + ) + negative_text_rope_pos = self._text_rope_pos( + batch.negative_attention_mask[0].to(prompt_embeds.device), + prompt_embeds.device, + ) + branches.append( + CFGBranch( + "unconditional", + False, + { + "encoder_hidden_states": neg_prompt_embeds, + "pooled_projections": neg_pooled, + "text_rope_pos": negative_text_rope_pos, + }, + ) + ) + return dataclasses.replace( + server_args.pipeline_config.cfg_policy, branches=branches + ) + + def _predict_joint_velocity( + self, + *, + transformer, + cfg_policy: CFGPolicy, + batch: Req, + server_args: ServerArgs, + step_index: int, + video_input: torch.Tensor, + audio_input: torch.Tensor, + t_expand: torch.Tensor, + visual_rope_pos: list[torch.Tensor], + scale_factor: tuple[float, ...], + sparse_params: dict[str, Any] | None, + visual_token_type_ids: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Run sequential or distributed CFG branches and combine video/audio velocity.""" + + def predict_fn(branch: CFGBranch) -> tuple[torch.Tensor, torch.Tensor]: + branch.configure_batch(batch) + with set_forward_context( + current_timestep=step_index, attn_metadata=None, forward_batch=batch + ): + model_output = self._call_transformer( + transformer, + hidden_states=video_input, + hidden_states_audio=audio_input, + timestep=t_expand, + visual_rope_pos=visual_rope_pos, + scale_factor=scale_factor, + sparse_params=sparse_params, + visual_token_type_ids=visual_token_type_ids, + # BCG clones tuple leaves so the next CFG replay cannot + # overwrite the previous branch's predictions + return_dict=False, + **branch.kwargs, + ) + return model_output + + cfg_scale = server_args.pipeline_config.get_classifier_free_guidance_scale( + batch, batch.guidance_scale + ) + + if server_args.enable_cfg_parallel: + if ( + len(cfg_policy.branches) == 2 + and get_classifier_free_guidance_world_size() == 2 + and not cfg_policy.parallel_uses_serial_arithmetic + ): + video_vel, audio_vel = run_two_branch_cfg_parallel( + cfg_policy, + predict_fn, + cfg_scale, + batch, + server_args.pipeline_config, + ) + return video_vel, audio_vel + predictions = run_cfg_parallel(cfg_policy, predict_fn) + else: + predictions = [predict_fn(branch) for branch in cfg_policy.branches] + + video_vel, audio_vel = cfg_policy.combine( + predictions, + batch, + cfg_scale, + server_args.pipeline_config, + cfg_parallel=server_args.enable_cfg_parallel, + ) + return video_vel, audio_vel + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + with self._offload_for_torch_compile_warmup(batch): + return self._run_joint_denoise_loop(batch, server_args) + + def _run_joint_denoise_loop(self, batch: Req, server_args: ServerArgs) -> Req: + if batch.timesteps is None: + raise ValueError("timesteps must be prepared before Kandinsky6 denoising.") + if batch.latents is None or batch.audio_latents is None: + raise ValueError( + "video and audio latents must be prepared before Kandinsky6 denoising." + ) + scheduler = batch.scheduler + if scheduler is None: + raise ValueError("scheduler must be set for Kandinsky6 denoising.") + use_piflow = bool(getattr(scheduler, "is_piflow", False)) + if use_piflow and ( + not math.isfinite(batch.guidance_scale) + or abs(batch.guidance_scale - 1.0) > 1e-6 + ): + raise ValueError("Kandinsky6 PiFlow requires guidance_scale=1.0.") + audio_scheduler = deepcopy(scheduler) if use_piflow else None + + pipeline_config = server_args.pipeline_config + arch = pipeline_config.dit_config.arch_config + # The runtime transformer already rejects attention_engine="nabla" + # at construction time (NABLA sparse attention is not yet ported), + # so sparse_params stays None on every currently-reachable + # configuration. A future NABLA-capable DiT would fill this in. + if arch.attention_engine == "nabla": + raise NotImplementedError( + "Kandinsky6 NABLA sparse-attention metadata construction is not implemented; " + "the runtime transformer also rejects attention_engine='nabla' at construction " + "time. Use attention_engine='auto' or 'sdpa'." + ) + sparse_params = None + + device = get_local_torch_device() + target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision] + + # .clone(): batch.latents was created by Kandinsky6LatentPreparationStage + # under an active torch.inference_mode() context further up the + # pipeline, which marks it as an "inference tensor" -- such tensors + # can only be mutated in place while still inside an inference_mode + # context. This stage runs under plain @torch.no_grad() instead, and + # mutates `video` in place every denoising step (video[..., :num_channels] + # = ...), so clone once here to get an ordinary, freely-mutable tensor + # before the loop starts. audio is never mutated in place (only + # reassigned via out-of-place `+`), so it needs no clone. + video = batch.latents.clone() + audio = batch.audio_latents + num_channels = int(arch.in_visual_dim) + + tail_cond = bool(batch.extra.get(TAIL_COND_ACTIVE_EXTRA_KEY, False)) + visual_token_type_ids = batch.extra.get(VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY) + + prompt_embeds = pipeline_config.get_pos_prompt_embeds(batch).to( + device=device, dtype=target_dtype + ) + if not batch.pooled_embeds: + raise ValueError("Kandinsky6 requires CLIP pooled projections.") + pooled = batch.pooled_embeds[0].to(device=device, dtype=target_dtype) + if not batch.prompt_attention_mask: + raise ValueError( + "Kandinsky6 requires Qwen (Reason1) prompt attention masks." + ) + text_rope_pos = self._text_rope_pos( + batch.prompt_attention_mask[0].to(device), device + ) + + cfg_policy = self._build_cfg_policy( + batch, + server_args, + prompt_embeds=prompt_embeds, + pooled=pooled, + text_rope_pos=text_rope_pos, + ) + + height = int(batch.height) + width = int(batch.width) + spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio + patch_size = arch.patch_size + + # Constant per-request geometry -- computed once outside the loop, + # not per-step. + num_video_frames = video.shape[1] - (1 if tail_cond else 0) + t_positions = torch.arange(num_video_frames, device=device) + if tail_cond: + # The appended reference frame reuses T-position 0's rope row + # (RoPE3D is a pure position -> table lookup, so duplicating the + # first row is equivalent to the diffusers reference's + # ``torch.cat([rope, rope[:1]])``). + t_positions = torch.cat([t_positions, t_positions.new_zeros(1)]) + visual_rope_pos = [ + t_positions, + torch.arange(height // spatial_ratio // patch_size[1], device=device), + torch.arange(width // spatial_ratio // patch_size[2], device=device), + ] + # Fixed per-checkpoint RoPE frequency scaling read from the DiT's + # arch config (transformer/config.json's "scale_factor", both real + # Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers + # reference, which resolves this once in + # ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every + # request regardless of the request's own height/width. NOT a + # function of the request's resolution. + scale_factor = arch.scale_factor + image_latent = batch.image_latent + + total_steps = int(batch.timesteps.shape[0]) + self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch) + + # PiFlow's noisy video state is tracked in fp32 across steps here, + # separately from the `video` buffer's bf16 storage dtype -- see the + # module docstring. Always a Tensor (not `Tensor | None`) even on the + # flow-Euler path, where it is simply unused: the initial cast is one + # cheap op, and keeping it non-Optional avoids re-narrowing it after + # every `if use_piflow:` branch re-entry in the loop below. + video_state = video[..., :num_channels].to(torch.float32) + + with self.use_declared_component( + component_name="transformer", module=self.transformer + ) as transformer: + assert transformer is not None + self.transformer = transformer + + with self.progress_bar( + total=total_steps, batch=batch, desc="Kandinsky6 Denoising" + ) as progress_bar: + for i, timestep in enumerate(batch.timesteps): + t_expand = ( + timestep.unsqueeze(0) + .repeat(video.shape[0]) + .to(device=device, dtype=target_dtype) + ) + + if use_piflow: + video_input = video_state.to(dtype=target_dtype) + if video.shape[-1] > num_channels: + video_input = torch.cat( + [ + video_input, + video[..., num_channels:].to(dtype=target_dtype), + ], + dim=-1, + ) + else: + video_input = video.to(dtype=target_dtype) + + video_vel, audio_vel = self._predict_joint_velocity( + transformer=transformer, + cfg_policy=cfg_policy, + batch=batch, + server_args=server_args, + step_index=i, + video_input=video_input, + audio_input=audio.to(dtype=target_dtype), + t_expand=t_expand, + visual_rope_pos=visual_rope_pos, + scale_factor=scale_factor, + sparse_params=sparse_params, + visual_token_type_ids=visual_token_type_ids, + ) + + # scheduler.step() is called exactly once per iteration, + # against video only -- it owns the internal step index. + if use_piflow: + video_state = scheduler.step( + video_vel, timestep, video_state, return_dict=False + )[0] + video[..., :num_channels] = video_state.to(video.dtype) + else: + video[..., :num_channels] = scheduler.step( + video_vel, + timestep, + video[..., :num_channels], + return_dict=False, + )[0] + if tail_cond: + ref_frame = image_latent.to( + device=video.device, dtype=video.dtype + ) + video[:, -1:, :, :, :num_channels] = ref_frame + if use_piflow: + video_state[:, -1:] = ref_frame.to(torch.float32) + + # Manual Euler update for audio, using the SAME per-step + # sigma delta the scheduler.step() call above just + # consumed for video. + if use_piflow: + audio = audio_scheduler.step( + audio_vel, timestep, audio, return_dict=False + )[0] + else: + step_size = scheduler.sigmas[i + 1] - scheduler.sigmas[i] + audio = ( + audio + + step_size.to(device=audio.device, dtype=audio.dtype) + * audio_vel + ) + + if progress_bar is not None: + progress_bar.update() + + video = video[..., :num_channels] + if tail_cond: + video = video[:, :-1] + + batch.latents = video + batch.audio_latents = audio + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check( + "audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)] + ) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.with_dims(1)]) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check( + "audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)] + ) + return result diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/image_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/image_encoding.py new file mode 100644 index 0000000000..3e6d5a6eb2 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/image_encoding.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional image conditioning, adapted from FastVideo's Kandinsky6ImageEncodingStage. + +Append a clean reference latent frame and mark its token type. Denoising re-pins +that frame every step and removes it before decoding.""" + +from __future__ import annotations + +import PIL.Image +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.condition_expansion import ( + PromptToSampleBatchExpander, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.precision import ( + autocast_context, + autocast_enabled, + resolve_precision, + temporary_module_dtype, +) +from sglang.multimodal_gen.runtime.utils.vision import ( + normalize, + numpy_to_pt, + pil_to_numpy, +) + +logger = init_logger(__name__) + +# model-specific reference-frame state shared through Req.extra +TAIL_COND_ACTIVE_EXTRA_KEY = "kandinsky6_tail_cond_active" +VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY = "kandinsky6_visual_token_type_ids" + + +class Kandinsky6ImageEncodingStage(PipelineStage): + """Optional IT2VA conditioning: no-op for a pure T2VA call. + + See module docstring for the tail-cond scheme. + """ + + def __init__(self, vae: ParallelTiledVAE) -> None: + super().__init__() + self.vae = vae + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + stage_name = self._component_stage_name(stage_name) + return [ComponentUse(stage_name, "vae", target_dtype=vae_dtype)] + + @staticmethod + def _cover_resize_dims( + src_h: int, src_w: int, height: int, width: int + ) -> tuple[int, int]: + """Resize to cover (height, width), preserving aspect before the centre crop.""" + scale = min(src_h / height, src_w / width) + # clamp float round-trip undershoot on the constraining axis before centre crop + new_h = max(height, int(src_h / scale)) + new_w = max(width, int(src_w / scale)) + return new_h, new_w + + @classmethod + def _preprocess(cls, image, height: int, width: int) -> torch.Tensor: + if isinstance(image, PIL.Image.Image): + src_w, src_h = image.size + new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width) + image = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR) + top, left = (new_h - height) // 2, (new_w - width) // 2 + image = image.crop((left, top, left + width, top + height)) + image = numpy_to_pt(pil_to_numpy(image)) + return normalize(image) + if image.min() >= 0: + logger.warning( + "Kandinsky6 conditioning image tensor has no negative values; " + "assuming range [0, 1] and normalizing to [-1, 1]. Pass a " + "[-1, 1] tensor with negative values to skip normalization." + ) + image = normalize(image) + if image.ndim == 3: + image = image.unsqueeze(0) + src_h, src_w = image.shape[-2], image.shape[-1] + if (src_h, src_w) != (height, width): + new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width) + image = torch.nn.functional.interpolate( + image.float(), size=(new_h, new_w), mode="bilinear", antialias=True + ) + top, left = (new_h - height) // 2, (new_w - width) // 2 + image = image[..., top : top + height, left : left + width] + return image + + @staticmethod + def _encode_scale_and_shift( + latents: torch.Tensor, scaling_factor, shift_factor + ) -> torch.Tensor: + if shift_factor is not None: + if isinstance(shift_factor, torch.Tensor): + shift_factor = shift_factor.to(latents.device, latents.dtype) + latents = latents - shift_factor + if isinstance(scaling_factor, torch.Tensor): + scaling_factor = scaling_factor.to(latents.device, latents.dtype) + return latents * scaling_factor + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + if batch.condition_image is None: + return batch + if batch.latents is None: + raise ValueError( + "Kandinsky6 video latents must be prepared before image conditioning." + ) + + image_source = batch.condition_image + if isinstance(image_source, list): + image_source = image_source[0] + + device = get_local_torch_device() + height = int(batch.height) + width = int(batch.width) + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast) + + image = self._preprocess(image_source, height, width) + image = image.to(device=device, dtype=torch.float32).unsqueeze(2) # [B,C,1,H,W] + + generator = batch.generator + if isinstance(generator, list) and len(generator) != image.shape[0]: + generator = generator[0] + + with self.use_declared_component(component_name="vae", module=self.vae) as vae: + assert vae is not None + self.vae = vae + prev_use_tiling = vae.use_tiling + vae.use_tiling = False + try: + with autocast_context(vae_dtype, server_args.disable_autocast): + should_cast_vae = not vae_autocast_enabled + if not vae_autocast_enabled: + image = image.to(vae_dtype) + with temporary_module_dtype( + vae, vae_dtype, enabled=should_cast_vae + ) as vae_cast: + image_latent = vae_cast.encode(image).sample( + generator=generator + ) + finally: + vae.use_tiling = prev_use_tiling + + scaling_factor, shift_factor = ( + server_args.pipeline_config.get_decode_scale_and_shift( + image_latent.device, image_latent.dtype, vae + ) + ) + + image_latent = self._encode_scale_and_shift( + image_latent, scaling_factor, shift_factor + ) + # [B,C,1,H,W] -> [B,1,H,W,C] channel-last, matching the video latent. + image_latent = image_latent.permute(0, 2, 3, 4, 1).contiguous() + batch.image_latent = image_latent + + latents = batch.latents + if batch.image_latent.shape[0] != latents.shape[0]: + # encode one reference image, then expand it to the request's output batch + expander = PromptToSampleBatchExpander( + prompt_batch_size=batch.image_latent.shape[0], + sample_batch_size=latents.shape[0], + ) + expander.expand_field(batch, "image_latent") + + image_latent = batch.image_latent.to(device=latents.device, dtype=latents.dtype) + num_channels = image_latent.shape[-1] + + ref_frame = image_latent + if latents.shape[-1] > num_channels: + # [real, cond, mask]: write the reference only into real; cond stays zero + cond_block = torch.zeros_like(image_latent) + mask_block = torch.ones_like(image_latent[..., :1]) + ref_frame = torch.cat([image_latent, cond_block, mask_block], dim=-1) + + batch.latents = torch.cat([latents, ref_frame], dim=1) + batch.extra[TAIL_COND_ACTIVE_EXTRA_KEY] = True + + num_video_frames = latents.shape[1] + token_type_ids = torch.zeros( + (latents.shape[0], num_video_frames + 1), dtype=torch.long, device=device + ) + token_type_ids[:, -1] = 1 + batch.extra[VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY] = token_type_ids + + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + return result diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/latent_preparation.py new file mode 100644 index 0000000000..9c3e6d8425 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6/latent_preparation.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Joint noise preparation, adapted from FastVideo's Kandinsky6LatentPreparationStage. + +Draw video then audio from the same generator. Image conditioning is appended later.""" + +from __future__ import annotations + +import math + +import torch +from diffusers.utils.torch_utils import randn_tensor + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE + + +def audio_latent_duration( + video_latent_frames: int, + *, + fps: float, + audio_sample_rate: int, + audio_downsample_factor: int, +) -> int: + """Audio latent length from causal video duration at the request's fps.""" + pixel_frames = (video_latent_frames - 1) * 4 + 1 + return int( + math.ceil(pixel_frames / fps * audio_sample_rate / audio_downsample_factor) + ) + + +class Kandinsky6LatentPreparationStage(PipelineStage): + """Draw initial video *and* audio noise latents.""" + + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + if batch.height is None or batch.width is None: + raise ValueError("height and width must be provided for Kandinsky6.") + height = int(batch.height) + width = int(batch.width) + num_frames = int(batch.num_frames) + + pipeline_config = server_args.pipeline_config + arch = pipeline_config.dit_config.arch_config + temporal_ratio = ( + pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio + patch_size = arch.patch_size + + if num_frames % temporal_ratio != 1: + num_frames = num_frames // temporal_ratio * temporal_ratio + 1 + batch.num_frames = num_frames + + required_divisor_h = spatial_ratio * patch_size[1] + required_divisor_w = spatial_ratio * patch_size[2] + if height % required_divisor_h != 0 or width % required_divisor_w != 0: + raise ValueError( + f"Kandinsky6 height must be divisible by {required_divisor_h} and width by " + f"{required_divisor_w}; got height={height}, width={width}." + ) + + batch_size = batch.batch_size + dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision] + device = get_local_torch_device() + num_latent_frames = (num_frames - 1) // temporal_ratio + 1 + num_channels = arch.in_visual_dim + video_shape = ( + batch_size, + num_latent_frames, + height // spatial_ratio, + width // spatial_ratio, + num_channels, + ) + + generator = batch.generator + if batch.latents is None: + video = randn_tensor( + video_shape, generator=generator, device=device, dtype=dtype + ) + else: + video = batch.latents.to(device=device, dtype=dtype) + if tuple(video.shape) != video_shape: + raise ValueError( + f"Provided latents shape {list(video.shape)} does not match expected " + f"Kandinsky6 video latent shape {list(video_shape)}." + ) + + if bool(arch.visual_cond): + cond = torch.zeros_like(video) + mask = torch.zeros( + (*video.shape[:-1], 1), device=video.device, dtype=video.dtype + ) + video = torch.cat([video, cond, mask], dim=-1) + + num_audio_channels = arch.in_audio_dim + # align audio duration with request fps, not the pipeline default + num_audio_frames = audio_latent_duration( + num_latent_frames, + fps=float(batch.fps), + audio_sample_rate=pipeline_config.audio_sample_rate, + audio_downsample_factor=pipeline_config.audio_downsample_factor, + ) + audio_shape = (batch_size, num_audio_frames, num_audio_channels) + if batch.audio_latents is None: + # Drawn from the same generator right after the video noise, so + # it is automatically an independent sample -- no manual seed + # offset needed. + audio = randn_tensor( + audio_shape, generator=generator, device=device, dtype=dtype + ) + else: + audio = batch.audio_latents.to(device=device, dtype=dtype) + if tuple(audio.shape) != audio_shape: + raise ValueError( + f"Provided audio_latents shape {list(audio.shape)} does not match expected " + f"Kandinsky6 audio latent shape {list(audio_shape)}." + ) + + batch.latents = video + batch.audio_latents = audio + batch.raw_latent_shape = ( + batch_size, + num_channels, + num_latent_frames, + height // spatial_ratio, + width // spatial_ratio, + ) + return batch + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("num_frames", batch.num_frames, V.positive_int) + result.add_check("height", batch.height, V.positive_int) + result.add_check("width", batch.width, V.positive_int) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) + result.add_check( + "audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)] + ) + return result diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py new file mode 100644 index 0000000000..f21295aef7 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers. + +Import the stage classes from their modules (``input_stage``, ``encode_stage``, +``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package +``__init__`` stays empty so that configs can import the pure helpers (``video_utils``, +``tiling``, ...) cheaply. +""" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py new file mode 100644 index 0000000000..0a0237a42a --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/decode_stage.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles.""" + +import torch + +from sglang.multimodal_gen.runtime.distributed import ( + get_decode_parallel_group_coordinator, + get_local_torch_device, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.vaes.common import ( + can_install_spatial_shard_parallel_decode, + has_decode_parallel_world, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_DENOISED_KEY, + SR_TILES_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + decode_chunks, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision import resolve_precision + + +class Kandinsky6SRDecodeStage(PipelineStage): + """VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``).""" + + def __init__(self, vae) -> None: + super().__init__() + self.vae = vae + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "vae", + phase="decode_tiles", + target_dtype=vae_dtype, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + denoised = batch.extra.pop(SR_DENOISED_KEY) + device = get_local_torch_device() + group = None + if ( + server_args.pipeline_config.vae_config.use_parallel_tiling + and has_decode_parallel_world() + and not can_install_spatial_shard_parallel_decode( + server_args.pipeline_config.vae_config + ) + ): + group = get_decode_parallel_group_coordinator() + with self.use_declared_component( + component_name="vae", module=self.vae, phase="decode_tiles" + ) as vae: + assert vae is not None + self.vae = vae + tiles = decode_chunks( + denoised, + vae, + scaling_factor=vae.scaling_factor, + device=device, + group=group, + ) + batch.extra[SR_TILES_KEY] = tiles + return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py new file mode 100644 index 0000000000..ce48fb2e8c --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/denoising_stage.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks. + +Each chunk resets its scheduler; the generic single-latent CFG loop and +prompt-padding BCG path do not apply.""" + +from __future__ import annotations + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_CHUNKS_KEY, + SR_DENOISED_KEY, + SR_DIT_SPEC_KEY, + SR_SAMPLING_SPEC_KEY, + effective_scheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + denoise_chunks, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + StageValidators as V, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + VerificationResult, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype + + +class Kandinsky6SRDenoisingStage(DenoisingStage): + """Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage.""" + + def __init__(self, transformer, scheduler, pipeline=None) -> None: + super().__init__( + transformer=transformer, + scheduler=scheduler, + pipeline=pipeline, + transformer_2=None, + vae=None, + ) + + def _owns_compile_warmup_lifecycle(self) -> bool: + # ``forward`` below does not go through the shared ``_denoise`` loop that this guard + # normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage): + # the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``. + return True + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + """Validate tile-chunk state instead of the shared text/CFG loop inputs.""" + result = VerificationResult() + result.add_check( + "extra[SR_SAMPLING_SPEC_KEY]", + batch.extra.get(SR_SAMPLING_SPEC_KEY), + V.not_none, + ) + result.add_check( + "extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none + ) + result.add_check( + "extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check( + "extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty + ) + return result + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + dit_dtype = precision_to_dtype( + server_args.pipeline_config.dit_precision, "dit_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "transformer", + phase="denoise_tiles", + target_dtype=dit_dtype, + preferred_ready_after_request=True, + memory_intensive=True, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + with self._offload_for_torch_compile_warmup(batch): + return self._denoise_tiles(batch, server_args) + + def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req: + spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY) + dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY) + chunks = batch.extra.pop(SR_CHUNKS_KEY) + device = get_local_torch_device() + + # Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the + # corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this + # request's DiT-call count, the closest analogue of the global ``num_inference_steps`` + # those hooks expect. + self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch) + + scheduler = effective_scheduler(spec, self.scheduler) + with self.use_declared_component( + component_name="transformer", module=self.transformer, phase="denoise_tiles" + ) as transformer: + assert transformer is not None + self.transformer = transformer + total = spec.steps_per_chunk * len(chunks) + with self.progress_bar( + total=total, batch=batch, desc="Kandinsky6 SR denoising" + ) as progress: + denoised = denoise_chunks( + chunks, + transformer, + scheduler, + dit_spec=dit_spec, + spec=spec, + device=device, + step_context=lambda step: set_forward_context( + current_timestep=step, attn_metadata=None, forward_batch=batch + ), + on_step=progress.update, + ) + self._finish_active_component_use() + batch.extra[SR_DENOISED_KEY] = denoised + return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py new file mode 100644 index 0000000000..97c554f1ef --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/encode_stage.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR.""" + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_LR_LATENT_KEY, + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, + uses_latent_path, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + encode_video_to_lr_latent, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision import resolve_precision + + +class Kandinsky6SREncodeStage(PipelineStage): + """Encode the whole clip only for the LU path; pixel mode encodes tiles later.""" + + def __init__(self, vae, latent_upscaler=None) -> None: + super().__init__() + self.vae = vae + # Only consulted to pick the path (does the bank have this scale?). + self.latent_upscaler = latent_upscaler + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + return [ + ComponentUse( + self._component_stage_name(stage_name), + "vae", + phase="encode_video", + target_dtype=vae_dtype, + start_at_stage_entry=False, + ) + ] + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + tiling_scale = batch.extra[SR_TILING_SCALE_KEY] + if not uses_latent_path(self.latent_upscaler, tiling_scale): + return batch + video = batch.extra.pop(SR_VIDEO_KEY) + with self.use_declared_component( + component_name="vae", module=self.vae, phase="encode_video" + ) as vae: + assert vae is not None + self.vae = vae + lr_latent = encode_video_to_lr_latent( + video, vae, device=get_local_torch_device() + ) + batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu() + return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py new file mode 100644 index 0000000000..6bb2936b9f --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/input_stage.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio).""" + +import os + +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_REQUESTED_HW_KEY, + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + VAE_SPATIAL_FACTOR, + pad_to_spatial_factor, + pre_upscale_video, + resolve_scale_request, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( + SOURCE_AUDIO_SAMPLE_RATE, + DecodedClip, + decode_clip, + extract_source_audio, + synthetic_clip, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + +# Warmup requests carry no (usable) video: run the real path on a small synthetic clip. +WARMUP_CLIP_FRAMES = 9 +WARMUP_CLIP_HW = (256, 384) + + +class Kandinsky6SRInputStage(PipelineStage): + """Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio. + + Record requested dimensions before VAE padding so output can remove it.""" + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale) + clip = self._load_clip(batch) + video = clip.frames + if pre_upscale != 1.0: + video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR) + frames = video.shape[0] + # Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel + # pre-upscale happens to already land on an aligned size): without this, a source + # whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4, + # which runs no pixel resize at all) fails partway through the whole-video KVAE + # encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps + # every source pixel; batch.height / batch.width below, and the output stage's final + # crop, restore the originally requested dimensions. + video, (requested_h, requested_w) = pad_to_spatial_factor( + video, VAE_SPATIAL_FACTOR + ) + + batch.extra[SR_VIDEO_KEY] = video + batch.extra[SR_TILING_SCALE_KEY] = tiling_scale + batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w) + batch.fps = clip.fps + batch.num_frames = frames + batch.height = requested_h * tiling_scale + batch.width = requested_w * tiling_scale + self._attach_source_audio(batch, frames=frames, fps=clip.fps) + return batch + + def _load_clip(self, batch: Req) -> DecodedClip: + if batch.is_warmup: + height, width = WARMUP_CLIP_HW + frames = synthetic_clip( + height=height, width=width, frames=WARMUP_CLIP_FRAMES + ) + return DecodedClip(frames=frames, fps=24, source_fps=24.0) + path = batch.sampling_params.source_video_path() + if path is None or not os.path.isfile(path): + raise ValueError(f"Kandinsky6 SR input video not found: {path!r}") + self.log_info("Decoding %s", path) + return decode_clip(path) + + @staticmethod + def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None: + if batch.is_warmup: + return + path = batch.sampling_params.source_video_path() + audio = extract_source_audio( + path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps + ) + if audio is None: + batch.audio = None + batch.audio_sample_rate = None + return + batch.audio = torch.from_numpy(audio) + batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py new file mode 100644 index 0000000000..c87180e817 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latent_prep_stage.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback. + +Validate sampler/head compatibility and step count before processing tiles.""" + +from functools import partial + +import torch + +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_CHUNKS_KEY, + SR_DIT_SPEC_KEY, + SR_LR_LATENT_KEY, + SR_PLAN_KEY, + SR_SAMPLING_SPEC_KEY, + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, + build_dit_spec, + build_sampling_spec, + check_denoising_request, + uses_latent_path, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + DitSpec, + SamplingSpec, + module_dtype, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + TilePlan, + plan_tiles, + prepare_lu_tile_latents, + prepare_pixel_tile_latents, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + VAE_SPATIAL_FACTOR, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.precision import ( + resolve_precision, +) + + +class Kandinsky6SRLatentPrepStage(PipelineStage): + """LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time.""" + + def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None: + super().__init__() + self.vae = vae + self.transformer = transformer # arch-config / dtype only: never called here + self.latent_upscaler = latent_upscaler + self.scheduler = ( + scheduler # resolved sampler validation only: never stepped here + ) + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + stage_name = self._component_stage_name(stage_name) + vae_dtype = resolve_precision( + server_args, "vae", precision_attr="vae_precision" + ) + uses: list[ComponentUse] = [] + if self.latent_upscaler is not None: + uses.append( + ComponentUse( + stage_name, + "latent_upscaler", + phase="upscale_tiles", + target_dtype=torch.bfloat16, + ) + ) + uses.append( + ComponentUse( + stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype + ) + ) + return uses + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + arch = server_args.pipeline_config.dit_config.arch_config + tiling_scale = batch.extra[SR_TILING_SCALE_KEY] + seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0] + spec = build_sampling_spec( + arch=arch, + tiling_scale=tiling_scale, + seed=seed, + num_steps=batch.num_inference_steps, + tiles_batch_size=batch.sr_tiles_batch_size, + tile_min_overlap=batch.sr_tile_min_overlap, + scheduler=self.scheduler, + ) + check_denoising_request(spec, batch.num_inference_steps) + dit_spec = build_dit_spec(self.transformer) + device = get_local_torch_device() + + plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device) + batch.extra[SR_PLAN_KEY] = plan + batch.extra[SR_CHUNKS_KEY] = chunks + batch.extra[SR_SAMPLING_SPEC_KEY] = spec + batch.extra[SR_DIT_SPEC_KEY] = dit_spec + return batch + + def _prepare_chunks( + self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device + ) -> tuple[TilePlan, list[torch.Tensor]]: + common = dict( + visual_size=spec.visual_size, + tiling_scale=spec.tiling_scale, + tile_min_overlap=spec.tile_min_overlap, + ) + if uses_latent_path(self.latent_upscaler, spec.tiling_scale): + lr_latent = batch.extra.pop(SR_LR_LATENT_KEY) + plan = plan_tiles( + frame_hw=( + lr_latent.shape[-2] * VAE_SPATIAL_FACTOR, + lr_latent.shape[-1] * VAE_SPATIAL_FACTOR, + ), + **common, + ) + with self.use_declared_component( + component_name="latent_upscaler", + module=self.latent_upscaler, + phase="upscale_tiles", + ) as bank: + assert bank is not None + self.latent_upscaler = bank + chunks = prepare_lu_tile_latents( + lr_latent, + plan, + upscale_fn=partial(bank.upscale, scale=spec.tiling_scale), + lu_dtype=module_dtype(bank), + scaling_factor=self.vae.scaling_factor, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + return plan, chunks + + video = batch.extra.pop(SR_VIDEO_KEY) + plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common) + with self.use_declared_component( + component_name="vae", module=self.vae, phase="encode_tiles" + ) as vae: + assert vae is not None + self.vae = vae + chunks = prepare_pixel_tile_latents( + video, + plan, + vae=vae, + scaling_factor=vae.scaling_factor, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + return plan, chunks diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py new file mode 100644 index 0000000000..297fcdb4d6 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/latents.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# Vendored from sr_core (parity-tested against the k6_video reference); adapted from +# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0). +"""SR latent construction with reference RNG ordering. + +Initial noise uses a per-request generator; conditioning noise uses the global RNG.""" + +from __future__ import annotations + +from typing import Literal + +import torch + +InstructType = Literal["noise", "channel", "hybrid", "hybrid_anchor"] +LqNoiseType = Literal["linear", "ddpm"] + + +def degrade_lq_latent( + lq_latent: torch.Tensor, + noise_scale: float = 0.7, + noise_type: LqNoiseType = "linear", + generator: torch.Generator | None = None, +) -> torch.Tensor: + """Mix linear or variance-preserving (ddpm) noise; nonpositive scales are a no-op.""" + if noise_scale <= 0: + return lq_latent + eps = torch.randn( + lq_latent.shape, + device=lq_latent.device, + dtype=lq_latent.dtype, + generator=generator, + ) + if noise_type == "ddpm": + return (1 - noise_scale**2) ** 0.5 * lq_latent + noise_scale * eps + return (1 - noise_scale) * lq_latent + noise_scale * eps + + +def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor: + """Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype).""" + if dtype is not None and value.is_floating_point() and value.dtype != dtype: + return value.to(dtype=dtype) + return value + + +def build_initial_latent( # noqa: PLR0913 + *, + instruct_type: InstructType, + visual_cond: bool, + in_visual_dim: int, + lq_latent: torch.Tensor, + batch_size: int, + duration: int, + height: int, + width: int, + device: str | int | torch.device, + seed: int, + lq_noise_scale: float, + lq_noise_type: LqNoiseType, + lq_channel_noise_scale: float, + dtype: torch.dtype | None = None, +) -> torch.Tensor: + """Build [starting | conditioning | mask] for the selected instruction mode. + + Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except + unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise.""" + lq_latent = lq_latent.to(device) + lq_latent = _cast_floating(lq_latent, dtype) + + if instruct_type == "noise": + g_noise = torch.Generator(device=device) + g_noise.manual_seed(seed) + degraded_lq = degrade_lq_latent( + lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise + ) + if not visual_cond: + # No conditioning channels in the model input layer (e.g. the + # t2v-initialized KVAE SR DiT) — feed the bare C-channel latent. + return degraded_lq + zero_cond = torch.zeros_like(degraded_lq) + zero_mask = torch.zeros( + [*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device + ) + return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1) + + if instruct_type in ("channel", "hybrid"): + if instruct_type == "hybrid": + g_noise = torch.Generator(device=device) + g_noise.manual_seed(seed) + starting = degrade_lq_latent( + lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise + ) + else: + g = torch.Generator(device=device) + g.manual_seed(seed) + starting = torch.randn( + batch_size * duration, + height, + width, + in_visual_dim, + device=device, + generator=g, + ) + channel_lq = degrade_lq_latent( + lq_latent, lq_channel_noise_scale, noise_type="linear" + ) + mask = torch.ones_like(lq_latent[..., :1]) + return torch.cat([starting, channel_lq, mask], dim=-1) + + if instruct_type == "hybrid_anchor": + # Anchor-free inference (tiled real-world LQ has no HR ground truth): + # a zeroed anchor + zeroed mask runs the model with no anchor signal, + # equivalent to anchor_indices=[] for the packed-latent datasets. + anchor_latent = torch.zeros_like(lq_latent) + anchor_mask = torch.zeros_like(lq_latent[..., :1]) + g_noise = torch.Generator(device=device) + g_noise.manual_seed(seed) + starting = degrade_lq_latent( + lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise + ) + anchor = degrade_lq_latent( + anchor_latent, lq_channel_noise_scale, noise_type="linear" + ) + return torch.cat([starting, anchor, anchor_mask], dim=-1) + + msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}" + raise ValueError(msg) + + +__all__ = [ + "InstructType", + "LqNoiseType", + "build_initial_latent", + "degrade_lq_latent", +] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py new file mode 100644 index 0000000000..6b1e8e5180 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/output_stage.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Stitch / crop / resize / output stage of Kandinsky 6 video SR.""" + +import torch + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( + resolve_target_hw, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_PLAN_KEY, + SR_REQUESTED_HW_KEY, + SR_TILES_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + stitch_tiles, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + crop_to_hw, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( + to_output_video, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( + resize_to_target, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class Kandinsky6SROutputStage(PipelineStage): + """Hann-blends the decoded tiles, restores the requested output size, applies the + optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1]) + plus the mono source audio. + + CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set + to the final size, ``batch.fps`` was set by the input stage (and now also carried on + the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save + path unchanged). + """ + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: + tiles = batch.extra.pop(SR_TILES_KEY) + plan = batch.extra.pop(SR_PLAN_KEY) + video = stitch_tiles(tiles, plan) + del tiles + + requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None) + if requested_hw is not None: + requested_h, requested_w = requested_hw + video = crop_to_hw( + video, + (requested_h * plan.tiling_scale, requested_w * plan.tiling_scale), + ) + + target_hw = resolve_target_hw( + batch.sr_target_resolution, + tuple(video.shape[-2:]), + batch.sr_target_resize_mode, + ) + if target_hw is not None: + video = resize_to_target(video, target_hw) + batch.height, batch.width = video.shape[-2:] + return OutputBatch( + output=to_output_video(video), + audio=batch.audio, + audio_sample_rate=batch.audio_sample_rate, + fps=batch.fps, + metrics=batch.metrics, + usage=batch.usage, + ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py new file mode 100644 index 0000000000..13711d5430 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/run_spec.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR request state and sampler resolution. + +batch.extra contains only CPU tensors and serializable data, never model objects.""" + +from typing import Any + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRArchConfig, +) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + DitSpec, + PiflowParams, + SamplingSpec, + module_dtype, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +SR_VIDEO_KEY = ( + "kandinsky6_sr_video" # [T, 3, H, W] uint8 clip (after pre-upscale + pad) +) +SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4) +SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path) +SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan +SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb] +# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out +# of the single combined stage so the denoising stage can subclass the shared DenoisingStage +# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain +# data, like TilePlan above), and the chunk lists are CPU tensors throughout. +SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec +SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec +SR_CHUNKS_KEY = ( + "kandinsky6_sr_chunks" # list of fp32 [B, T', H', W', 2C+1] (pre-denoise) +) +SR_DENOISED_KEY = ( + "kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise) +) +# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the +# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale +# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The +# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to +# (H * scale, W * scale) so the delivered video matches what was actually asked for. +SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw" + + +def uses_latent_path(latent_upscaler: Any, tiling_scale: int) -> bool: + """LU path iff the bank has an entry for the tiling scale (else the pixel path).""" + return ( + latent_upscaler is not None + and latent_upscaler.for_scale(tiling_scale) is not None + ) + + +def build_dit_spec(dit: Any) -> DitSpec: + """Resolved DiT input and conditioning layout.""" + return DitSpec( + instruct_type=dit.instruct_type, + visual_cond=bool(dit.visual_cond), + in_visual_dim=dit.in_visual_dim, + use_motion_score=bool(dit.use_motion_score), + patch_size=tuple(dit.patch_size), + dtype=module_dtype(dit), + ) + + +def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]: + table = arch.sr_scale_factor + visual_size = arch.sr_visual_size[0] + if table is None: + raise ValueError( + "transformer/config.json carries no sr_params (visual_size / scale_factor); " + "an official Diffusers Kandinsky6SRPipeline repo is required" + ) + values = table.get(str(visual_size), table.get(visual_size)) + if values is None: + raise ValueError( + f"sr_scale_factor has no entry for visual size {visual_size}: {table}" + ) + return tuple(float(value) for value in values) + + +def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None: + """pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler).""" + if not isinstance(scheduler, PiflowScheduler): + return None + config = scheduler.config + if config["nfe"] is None: + return None + return PiflowParams( + nfe=int(config["nfe"]), + num_policy_substeps=int(config["num_policy_substeps"]), + final_step_size_scale=float(config["final_step_size_scale"]), + shift=float(config["shift"]), + n_grid=int(config["n_grid"]), + eps=float(config["eps"]), + ) + + +def _describe_scheduler(scheduler: Any) -> str: + if scheduler is None: + return "missing" + if isinstance(scheduler, PiflowScheduler): + return "a PiflowScheduler without nfe" + return f"a {type(scheduler).__name__}" + + +def _check_sampler_fits_head( + *, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any +) -> None: + """Reject incompatible scheduler and DiT head widths before encoding tiles.""" + head, grid = arch.head_width, arch.base_out_visual_dim + if piflow is None: + if head != grid: + raise ValueError( + f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / " + f"pi-Flow head: {grid} channels per latent), but the sampler resolved " + "to flow-Euler: the scheduler component is " + f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with " + "nfe selects pi-Flow. Use the scheduler of the same repo as the " + "transformer." + ) + return + if piflow.nfe < 1: + raise ValueError( + "Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, " + f"got nfe={piflow.nfe}." + ) + if head != piflow.n_grid * grid: + raise ValueError( + f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid " + f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = " + f"{piflow.n_grid * grid} channel DX head, but the transformer head is " + f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The " + "scheduler and the transformer come from different repos." + ) + + +def build_sampling_spec( + *, + arch: Kandinsky6SRArchConfig, + tiling_scale: int, + seed: int, + num_steps: int, + tiles_batch_size: int, + tile_min_overlap: float, + scheduler: Any = None, +) -> SamplingSpec: + """Resolve sampling from the bundle scheduler, falling back to legacy DiT fields. + + num_steps counts DiT calls, not Diffusers timestep grid points.""" + piflow = piflow_params_from_scheduler(scheduler) + if piflow is None and arch.is_piflow: + piflow = PiflowParams( + nfe=arch.piflow_nfe, + num_policy_substeps=arch.piflow_num_policy_substeps, + final_step_size_scale=arch.piflow_final_step_size_scale, + shift=arch.piflow_shift, + n_grid=arch.n_grid, + eps=arch.piflow_eps, + ) + _check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler) + return SamplingSpec( + tiling_scale=tiling_scale, + tiles_batch_size=tiles_batch_size, + seed=seed, + num_steps=num_steps, + tile_min_overlap=tile_min_overlap, + visual_size=arch.sr_visual_size[0], + scale_factor=_rope_scale_factor(arch), + scheduler_scale=arch.sr_scheduler_scale, + lq_noise_scale=arch.sr_lq_noise_scale, + lq_noise_type=arch.sr_lq_noise_type, + lq_channel_noise_scale=arch.sr_lq_channel_noise_scale, + cap_noise_timestep=arch.sr_cap_noise_timestep, + piflow=piflow, + ) + + +def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any: + """Use the loaded scheduler when compatible; synthesize one for legacy configs.""" + if spec.piflow is not None: + if isinstance(scheduler, PiflowScheduler): + return scheduler + return PiflowScheduler( + nfe=spec.piflow.nfe, + n_grid=spec.piflow.n_grid, + shift=spec.piflow.shift, + eps=spec.piflow.eps, + final_step_size_scale=spec.piflow.final_step_size_scale, + num_policy_substeps=spec.piflow.num_policy_substeps, + ) + if isinstance(scheduler, FlowMatchEulerDiscreteScheduler): + return scheduler + return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale) + + +def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None: + """Validate the requested DiT-call count before denoising. + + Diffusers counts timestep grid points instead: its default 5 is 4 calls here.""" + if num_inference_steps < 1: + raise ValueError( + f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}" + ) + if spec.piflow is not None and num_inference_steps != spec.piflow.nfe: + logger.warning( + "Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow " + "bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow " + "bundles and every tile will run exactly nfe=%d DiT calls instead.", + num_inference_steps, + spec.piflow.nfe, + spec.piflow.nfe, + ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py new file mode 100644 index 0000000000..b6d7287558 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/sampling.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Scheduler-driven denoising for Kandinsky 6 SR tile chunks.""" + +from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext +from typing import Any + +import msgspec +import torch + +# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output +DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor] +StepCallback = Callable[[], None] +StepContext = Callable[[int], AbstractContextManager] + + +class PiflowParams(msgspec.Struct, frozen=True, kw_only=True): + """pi-Flow parameters from the scheduler or legacy flat DiT config.""" + + nfe: int + num_policy_substeps: int + final_step_size_scale: float + shift: float + n_grid: int + eps: float + + +class DitSpec(msgspec.Struct, frozen=True, kw_only=True): + """Post-override behaviour of the DiT that the sampling code needs.""" + + instruct_type: str | None + visual_cond: bool + in_visual_dim: int + use_motion_score: bool + patch_size: tuple[int, int, int] + # The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the + # LQ latent to this dtype before building the initial latent, so a bf16-loaded + # checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT, + # e.g. a mocked one in a test) keeps the LQ latent's own dtype. + dtype: torch.dtype | None = None + + +class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True): + """Everything besides the components that defines one tiled SR run.""" + + tiling_scale: int + tiles_batch_size: int + seed: int + num_steps: int + tile_min_overlap: float + visual_size: int + scale_factor: tuple[float, float, float] + scheduler_scale: float + lq_noise_scale: float + lq_noise_type: str + lq_channel_noise_scale: float + cap_noise_timestep: bool + piflow: PiflowParams | None + + @property + def steps_per_chunk(self) -> int: + """DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``.""" + return self.piflow.nfe if self.piflow is not None else self.num_steps + + +def bf16_autocast(device: torch.device | str) -> AbstractContextManager: + """CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models.""" + if torch.device(device).type != "cuda": + return nullcontext() + return torch.autocast(device_type="cuda", dtype=torch.bfloat16) + + +def module_dtype(module: torch.nn.Module) -> torch.dtype | None: + """Dtype of the first parameter (``None`` for a parameter-less module).""" + return next((param.dtype for param in module.parameters()), None) + + +def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor: + """Cast a floating tensor to the module's parameter dtype (reference helper).""" + dtype = module_dtype(module) + if dtype is not None and value.is_floating_point() and value.dtype != dtype: + return value.to(dtype=dtype) + return value + + +def euler_start_timestep( + *, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None +) -> float: + """Euler start time: the LQ noise level when capped for noise-type conditioning.""" + capped = cap_noise_timestep and instruct_type in ( + "noise", + "hybrid", + "hybrid_anchor", + ) + return lq_noise_scale if capped else 1.0 + + +def check_sampler_options( + *, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None +) -> None: + """Reject option combinations the reference sampler rejects.""" + if ( + piflow is not None + and cap_noise_timestep + and instruct_type in ("noise", "hybrid") + ): + raise NotImplementedError( + "piflow sampler does not support cap_noise_timestep for noise/hybrid " + "instruct" + ) + + +def make_dit_fn( + dit: Callable[..., torch.Tensor], + *, + latent_frames_hw: tuple[int, int, int], + patch_size: tuple[int, int, int], + scale_factor: tuple[float, float, float], + use_motion_score: bool, + step_context: StepContext | None = None, +) -> DitFn: + """Bind per-chunk RoPE/motion inputs and an optional per-step context.""" + frames, height, width = latent_frames_hw + context = step_context or (lambda step: nullcontext()) + + def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor: + rope_pos = [ + torch.arange(frames, device=x.device), + torch.arange(height // patch_size[1], device=x.device), + torch.arange(width // patch_size[2], device=x.device), + ] + motion = ( + torch.full((1,), 900.0, device=x.device, dtype=x.dtype) + if use_motion_score + else None + ) + with context(step): + return dit( + x, + model_time, + rope_pos, + scale_factor=scale_factor, + motion_score=motion, + ) + + return call + + +@torch.no_grad() +def denoise_with_scheduler( + x: torch.Tensor, + dit_fn: DitFn, + scheduler: Any, + *, + channels: int, + is_piflow: bool, + on_step: StepCallback | None = None, +) -> torch.Tensor: + """Denoise one chunk after set_timesteps; keep conditioning channels fixed. + + Promote predictions to fp32 so flow-Euler does not downcast its running + state to the bf16 prediction dtype. pi-Flow already preserves the state dtype.""" + state = x[..., :channels].float() + cond = x[..., channels:] + for step, t in enumerate(scheduler.timesteps): + model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0]) + hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state + prediction = dit_fn(hidden_states, model_time, step) + if not is_piflow: + prediction = prediction.float() + state = scheduler.step(prediction, t, state, return_dict=False)[0] + if on_step is not None: + on_step() + return state.float() + + +__all__ = [ + "DitFn", + "DitSpec", + "PiflowParams", + "SamplingSpec", + "StepCallback", + "StepContext", + "bf16_autocast", + "cast_to_module_dtype", + "check_sampler_options", + "denoise_with_scheduler", + "euler_start_timestep", + "make_dit_fn", + "module_dtype", +] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py new file mode 100644 index 0000000000..3c801b1b60 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiled.py @@ -0,0 +1,413 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tile preparation, denoising and blending for Kandinsky 6 SR. + +Stages group work by component to avoid swapping weights per tile. +Each chunk uses seed + its first tile index, matching the reference. +""" + +from collections.abc import Callable, Iterator, Mapping, Sequence +from typing import Any + +import msgspec +import torch + +from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import ( + build_initial_latent, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + DitSpec, + SamplingSpec, + StepCallback, + StepContext, + bf16_autocast, + cast_to_module_dtype, + check_sampler_options, + denoise_with_scheduler, + euler_start_timestep, + make_dit_fn, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + RESOLUTIONS, + VAE_SPATIAL_FACTOR, + TileGrid, + closest_base_resolution, + compute_tile_grid_even, + extract_all_tiles, + latent_tile_grid_from_pixel_grid, + stitch_tiles_hanning, + upsample_tiles_to_base, +) + +LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor] +STITCH_FRAME_CHUNK = 8 + + +class TilePlan(msgspec.Struct, frozen=True): + """Tile geometry of one run (all sizes in pixels of the frame that is tiled).""" + + tiling_scale: int + frame_hw: tuple[int, int] + base_hw: tuple[int, int] + tile_hw: tuple[int, int] + pixel_grid: TileGrid + + +def plan_tiles( + *, + frame_hw: tuple[int, int], + visual_size: int, + tiling_scale: int, + tile_min_overlap: float, + spatial_factor: int = VAE_SPATIAL_FACTOR, + resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS, +) -> TilePlan: + """Even tile grid for a frame; the frame must hold at least one full tile.""" + height, width = frame_hw + base_hw = closest_base_resolution(height, width, visual_size, resolutions) + if any(size % tiling_scale for size in base_hw): + raise ValueError( + f"resolution_scale={tiling_scale} does not divide the base resolution " + f"{base_hw[0]}x{base_hw[1]} exactly" + ) + tile_hw = (base_hw[0] // tiling_scale, base_hw[1] // tiling_scale) + grid = compute_tile_grid_even( + height, width, tile_hw, tile_min_overlap, spatial_factor + ) + if height < tile_hw[0] or width < tile_hw[1]: + raise ValueError( + f"The {width}x{height} frame is smaller than one SR tile " + f"({tile_hw[1]}x{tile_hw[0]}) at x{tiling_scale}; use a larger input or a " + "higher scale." + ) + return TilePlan( + tiling_scale=tiling_scale, + frame_hw=(height, width), + base_hw=base_hw, + tile_hw=tile_hw, + pixel_grid=grid, + ) + + +def chunk_ranges(num_tiles: int, tiles_batch_size: int) -> Iterator[tuple[int, int]]: + """``(start, stop)`` tile ranges; ``start`` also offsets the chunk's seed.""" + for start in range(0, num_tiles, tiles_batch_size): + yield start, min(start + tiles_batch_size, num_tiles) + + +@torch.no_grad() +def encode_video_to_lr_latent( + video: torch.Tensor, vae: torch.nn.Module, *, device: torch.device +) -> torch.Tensor: + """Encode a whole ``[T, C, H, W]`` uint8 video to the raw latent ``[T', C', h, w]``. + + The latent is *not* multiplied by the VAE scaling factor (the LU does that itself). + """ + if video.ndim != 4: + raise ValueError(f"video must be [T, C, H, W], got {tuple(video.shape)}") + pixel = video.permute(1, 0, 2, 3).unsqueeze(0).to(device=device) + pixel = cast_to_module_dtype(vae, vae.normalize_data(pixel.float())) + latent = vae.encode(pixel)[0] + return latent.squeeze(0).permute(1, 0, 2, 3).float() + + +@torch.no_grad() +def upscale_lr_latent_tile( + tile: torch.Tensor, + upscale_fn: LatentUpscaleFn, + *, + lu_dtype: torch.dtype | None, + scaling_factor: float, + device: torch.device, +) -> torch.Tensor: + """LU on one raw tile ``[T', C', h, w]`` -> scaled latent ``[T', Hb, Wb, C']``.""" + z = tile.permute(1, 0, 2, 3).unsqueeze(0).to(device=device, dtype=torch.float32) + if lu_dtype is not None and z.dtype != lu_dtype: + z = z.to(dtype=lu_dtype) + with bf16_autocast(device): + upscaled = upscale_fn(z * scaling_factor) + return upscaled.squeeze(0).permute(1, 2, 3, 0).float() + + +@torch.no_grad() +def encode_pixel_tile( + tile: torch.Tensor, + vae: torch.nn.Module, + *, + scaling_factor: float, + device: torch.device, +) -> torch.Tensor: + """Pixel path: ``[T, Hb, Wb, 3]`` floats in [0, 255] -> scaled ``[T', Hb', Wb', C']``. + + The tile goes through bf16 first (also on CPU), like the reference. + """ + lq = tile.permute(3, 0, 1, 2).unsqueeze(0).to(device=device, dtype=torch.bfloat16) + lq = cast_to_module_dtype(vae, vae.normalize_data(lq)) + latent = vae.encode(lq)[0] + return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor + + +def build_chunk_latent( + lq_tiles: Sequence[torch.Tensor], + *, + seed: int, + dit_spec: DitSpec, + spec: SamplingSpec, + device: torch.device, +) -> torch.Tensor: + """Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles.""" + lq = torch.cat(list(lq_tiles), dim=0) + batch = len(lq_tiles) + x = build_initial_latent( + instruct_type=dit_spec.instruct_type, + visual_cond=dit_spec.visual_cond, + in_visual_dim=dit_spec.in_visual_dim, + lq_latent=lq, + batch_size=batch, + duration=lq.shape[0] // batch, + height=lq.shape[1], + width=lq.shape[2], + device=device, + seed=seed, + lq_noise_scale=spec.lq_noise_scale, + lq_noise_type=spec.lq_noise_type, + lq_channel_noise_scale=spec.lq_channel_noise_scale, + # The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before + # building the initial latent), not a hard-coded fp32 regardless of precision. + dtype=dit_spec.dtype, + ) + return x.reshape(batch, -1, *x.shape[1:]) + + +@torch.no_grad() +def prepare_lu_tile_latents( + lr_latent: torch.Tensor, + plan: TilePlan, + *, + upscale_fn: LatentUpscaleFn, + lu_dtype: torch.dtype | None, + scaling_factor: float, + dit_spec: DitSpec, + spec: SamplingSpec, + device: torch.device, + spatial_factor: int = VAE_SPATIAL_FACTOR, +) -> list[torch.Tensor]: + """LU path: initial latents (CPU) of every chunk from the whole-video latent.""" + latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor) + tiles = extract_all_tiles(lr_latent, latent_grid) + chunks: list[torch.Tensor] = [] + for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size): + lq_tiles = [ + upscale_lr_latent_tile( + tile, + upscale_fn, + lu_dtype=lu_dtype, + scaling_factor=scaling_factor, + device=device, + ) + for tile in tiles[start:stop] + ] + chunk = build_chunk_latent( + lq_tiles, + seed=spec.seed + start, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + chunks.append(chunk.cpu()) + return chunks + + +@torch.no_grad() +def prepare_pixel_tile_latents( + video: torch.Tensor, + plan: TilePlan, + *, + vae: torch.nn.Module, + scaling_factor: float, + dit_spec: DitSpec, + spec: SamplingSpec, + device: torch.device, +) -> list[torch.Tensor]: + """Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk.""" + tiles = extract_all_tiles(video, plan.pixel_grid) + chunks: list[torch.Tensor] = [] + for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size): + enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw) + lq_tiles = [ + encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device) + for tile in enlarged + ] + chunk = build_chunk_latent( + lq_tiles, + seed=spec.seed + start, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + chunks.append(chunk.cpu()) + return chunks + + +def reset_scheduler_for_chunk(scheduler: Any) -> None: + """Reset step and begin indices so each tile chunk starts at the same timestep.""" + scheduler._step_index = None + scheduler.set_begin_index(0) + + +@torch.no_grad() +def denoise_chunk( + x: torch.Tensor, + dit: Callable[..., torch.Tensor], + scheduler: Any, + *, + dit_spec: DitSpec, + spec: SamplingSpec, + step_context: StepContext | None = None, + on_step: StepCallback | None = None, +) -> torch.Tensor: + """Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler.""" + is_piflow = spec.piflow is not None + check_sampler_options( + piflow=spec.piflow, + cap_noise_timestep=spec.cap_noise_timestep, + instruct_type=dit_spec.instruct_type, + ) + if not is_piflow: + # The Euler start timestep is only meaningful for the scheduler this chunk is about to + # run: re-derive it instead of baking a stale one into ``set_timesteps`` up front. + start = euler_start_timestep( + cap_noise_timestep=spec.cap_noise_timestep, + lq_noise_scale=spec.lq_noise_scale, + instruct_type=dit_spec.instruct_type, + ) + sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist() + scheduler.set_timesteps(sigmas=sigmas, device=x.device) + reset_scheduler_for_chunk(scheduler) + dit_fn = make_dit_fn( + dit, + latent_frames_hw=tuple(x.shape[1:4]), + patch_size=dit_spec.patch_size, + scale_factor=spec.scale_factor, + use_motion_score=dit_spec.use_motion_score and not is_piflow, + step_context=step_context, + ) + with bf16_autocast(x.device): + return denoise_with_scheduler( + x, + dit_fn, + scheduler, + channels=dit_spec.in_visual_dim, + is_piflow=is_piflow, + on_step=on_step, + ) + + +def denoise_chunks( + chunks: Sequence[torch.Tensor], + dit: Callable[..., torch.Tensor], + scheduler: Any, + *, + dit_spec: DitSpec, + spec: SamplingSpec, + device: torch.device, + step_context: StepContext | None = None, + on_step: StepCallback | None = None, +) -> list[torch.Tensor]: + """Denoise chunks sequentially on device and return CPU latents.""" + if spec.piflow is not None: + scheduler.set_timesteps(spec.piflow.nfe, device=device) + return [ + denoise_chunk( + chunk.to(device), + dit, + scheduler, + dit_spec=dit_spec, + spec=spec, + step_context=step_context, + on_step=on_step, + ).cpu() + for chunk in chunks + ] + + +@torch.no_grad() +def _decode_tile( + latents: torch.Tensor, + vae: torch.nn.Module, + *, + scaling_factor: float, + device: torch.device, +) -> torch.Tensor: + """Decode one ``[1, T', H', W', C]`` tile to uint8 ``[3, T, H, W]`` on device.""" + with bf16_autocast(device): + z = (latents.to(device) / scaling_factor).permute(0, 4, 1, 2, 3) + latent = cast_to_module_dtype(vae, z) + # preserve the reference's FP32 rescale before uint8 quantization + decoded = vae.decode(latent).sample.float() + pixels = (vae.denormalize_data(decoded) / 255.0).clamp(0.0, 1.0) + return (pixels * 255.0).to(torch.uint8).squeeze(0) + + +def decode_chunks( + chunks: Sequence[torch.Tensor], + vae: torch.nn.Module, + *, + scaling_factor: float, + device: torch.device, + group: GroupCoordinator | None = None, +) -> list[torch.Tensor]: + """Decode in tile order, optionally sharing independent tiles within one DP replica.""" + latents = [tile for chunk in chunks for tile in chunk.split(1)] + if group is None or group.world_size == 1 or len(latents) == 1: + return [ + _decode_tile(tile, vae, scaling_factor=scaling_factor, device=device).cpu() + for tile in latents + ] + + tiles = [] + for start in range(0, len(latents), group.world_size): + index = start + group.rank_in_group + if index < len(latents): + decoded = _decode_tile( + latents[index], vae, scaling_factor=scaling_factor, device=device + ) + else: + _, frames, height, width, _ = latents[start].shape + decoded = torch.zeros( + ( + 3, + 1 + (frames - 1) * vae.temporal_factor, + height * vae.spatial_factor, + width * vae.spatial_factor, + ), + device=device, + dtype=torch.uint8, + ) + # bound device staging to one tile per rank, not the whole decoded video + gathered = group.all_gather(decoded.unsqueeze(0))[: len(latents) - start].cpu() + del decoded + tiles.extend(gathered.unbind(0)) + return tiles + + +@torch.no_grad() +def stitch_tiles(tiles: Sequence[torch.Tensor], plan: TilePlan) -> torch.Tensor: + """Hann-blend [C, T, H, W] tiles in frame chunks to bound memory.""" + height, width = plan.frame_hw + scale = plan.tiling_scale + channels, frames = tiles[0].shape[:2] + out = torch.empty( + (channels, frames, height * scale, width * scale), dtype=torch.uint8 + ) + for start in range(0, frames, STITCH_FRAME_CHUNK): + stop = min(start + STITCH_FRAME_CHUNK, frames) + blended = stitch_tiles_hanning( + [tile[:, start:stop].float() for tile in tiles], + plan.pixel_grid, + height, + width, + scale=scale, + ) + out[:, start:stop] = blended.clamp(0, 255).to(torch.uint8) + return out diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py new file mode 100644 index 0000000000..f523d225a0 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/tiling.py @@ -0,0 +1,248 @@ +# SPDX-License-Identifier: Apache-2.0 +# Vendored from sr_core (parity-tested against the k6_video reference); adapted from +# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0). +"""VAE-aligned tile grids and Hann blending for Kandinsky 6 video SR.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from typing import NamedTuple + +import torch +from torch.nn import functional + +VAE_SPATIAL_FACTOR: int = 16 +VAE_TEMPORAL_FACTOR: int = 4 + +# trained base resolutions: visual_size -> [(H, W), ...] +RESOLUTIONS: dict[int, list[tuple[int, int]]] = { + 512: [(512, 512), (512, 768), (768, 512)], +} + + +class TileGrid(NamedTuple): + """Tile starts are authoritative; gaps can differ near the frame edges.""" + + tile_h: int + tile_w: int + tops: tuple[int, ...] + lefts: tuple[int, ...] + + @property + def n_h(self) -> int: + """Number of tile rows.""" + return len(self.tops) + + @property + def n_w(self) -> int: + """Number of tile columns.""" + return len(self.lefts) + + @property + def total_tiles(self) -> int: + """Total number of tiles (``n_h * n_w``).""" + return self.n_h * self.n_w + + +def extract_all_tiles( + video: torch.Tensor, + grid: TileGrid, +) -> list[torch.Tensor]: + """Extract [T, C, tile_h, tile_w] views in row-major order.""" + tiles: list[torch.Tensor] = [] + for top in grid.tops: + for left in grid.lefts: + tile = video[:, :, top : top + grid.tile_h, left : left + grid.tile_w] + tiles.append(tile) + return tiles + + +def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """Normalize Hann-weighted tiles without flooring positive weights. + + Corner weights are tiny but valid; clamping them darkens tile boundaries. + Pixels outside all tiles have zero weight and remain zero.""" + covered = weight > 0 + safe_weight = torch.where(covered, weight, torch.ones_like(weight)) + return torch.where(covered, acc / safe_weight, torch.zeros_like(acc)) + + +def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor: + """Create a [h, w] Hann window with nonzero endpoints.""" + wy = torch.hann_window(h + 2, device=device)[1:-1] + wx = torch.hann_window(w + 2, device=device)[1:-1] + return wy[:, None] * wx[None, :] + + +def stitch_tiles_hanning( + tiles: list[torch.Tensor], + grid: TileGrid, + original_h: int, + original_w: int, + scale: int = 1, +) -> torch.Tensor: + """Blend row-major [C, T, H, W] tiles into a scale-enlarged frame.""" + first = tiles[0] + c, t = first.shape[0], first.shape[1] + hr_tile_h, hr_tile_w = first.shape[2], first.shape[3] + device = first.device + + window = hanning_window_2d(hr_tile_h, hr_tile_w, device) + window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w] + + out_h = original_h * scale + out_w = original_w * scale + pred_acc = torch.zeros(c, t, out_h, out_w, device=device) + weight_acc = torch.zeros(1, 1, out_h, out_w, device=device) + + tile_idx = 0 + for top in grid.tops: + for left in grid.lefts: + y = top * scale + x = left * scale + tile = tiles[tile_idx] + pred_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += tile * window + weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window + tile_idx += 1 + + return _normalize_blend(pred_acc, weight_acc) + + +def axis_positions_even( + length: int, tile: int, min_overlap: float, snap: int +) -> tuple[int, ...]: + """Cover the axis with the fewest evenly spaced, snap-aligned tiles. + + Fall back to pixel alignment for unaligned sizes; latent-grid conversion + validates VAE alignment separately.""" + if tile <= 0 or snap <= 0 or not 0 <= min_overlap < 1: + msg = f"invalid axis spec: tile={tile}, snap={snap}, min_overlap={min_overlap}" + raise ValueError(msg) + if tile >= length: + return (0,) + unit = snap if length % snap == 0 and tile % snap == 0 else 1 + span_units = (length - tile) // unit + max_stride_units = max(1, math.floor(tile * (1.0 - min_overlap) / unit)) + count = math.ceil(span_units / max_stride_units) + 1 + return tuple(round(i * span_units / (count - 1)) * unit for i in range(count)) + + +def compute_tile_grid_even( + h: int, + w: int, + tile_hw: tuple[int, int], + min_overlap: float, + snap: int, +) -> TileGrid: + """Build a grid with evenly distributed, aligned tile starts.""" + tile_h, tile_w = tile_hw + return TileGrid( + tile_h=tile_h, + tile_w=tile_w, + tops=axis_positions_even(h, tile_h, min_overlap, snap), + lefts=axis_positions_even(w, tile_w, min_overlap, snap), + ) + + +def latent_tile_grid_from_pixel_grid( + pixel_grid: TileGrid, spatial_factor: int +) -> TileGrid: + """Convert an aligned pixel grid to the corresponding latent grid.""" + values = (pixel_grid.tile_h, pixel_grid.tile_w, *pixel_grid.tops, *pixel_grid.lefts) + if any(value % spatial_factor for value in values): + raise ValueError( + f"Pixel tile grid (tile {pixel_grid.tile_h}x{pixel_grid.tile_w}, " + f"tops={pixel_grid.tops}, lefts={pixel_grid.lefts}) not aligned to VAE spatial factor " + f"{spatial_factor}." + ) + return TileGrid( + pixel_grid.tile_h // spatial_factor, + pixel_grid.tile_w // spatial_factor, + tuple(top // spatial_factor for top in pixel_grid.tops), + tuple(left // spatial_factor for left in pixel_grid.lefts), + ) + + +def closest_base_resolution( + h: int, + w: int, + visual_size: int, + resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS, +) -> tuple[int, int]: + """Choose the trained base resolution with the closest aspect ratio.""" + if visual_size not in resolutions: + raise ValueError( + f"Unsupported SR visual_size={visual_size}; known sizes: {sorted(resolutions)}" + ) + ratio = w / h if h else 1.0 + return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio)) + + +def upsample_tiles_to_base( + raw_tiles: list[torch.Tensor], base_h: int, base_w: int +) -> list[torch.Tensor]: + """Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``.""" + return [ + functional.interpolate( + tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False + ).permute(0, 2, 3, 1) + for tile in raw_tiles + ] + + +def resolve_scale_request(scale: float) -> tuple[int, float]: + """Decompose total scale into (integer tiling scale, pixel pre-upscale).""" + requested = float(scale) + if requested == 2.25: # the one supported fractional total + return 2, 1.125 + if requested in (2.0, 4.0): + return int(requested), 1.0 + raise ValueError("SR supports total scales 2, 4, and 2.25") + + +def pre_upscale_video( + video: torch.Tensor, factor: float, spatial_multiple: int +) -> torch.Tensor: + """Bilinearly enlarge [T, C, H, W] uint8 video, aligning target dimensions.""" + if video.ndim != 4: + raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}") + if factor <= 0 or spatial_multiple <= 0: + raise ValueError("factor and spatial_multiple must be positive") + height, width = video.shape[-2:] + target_h = max( + spatial_multiple, round(height * factor / spatial_multiple) * spatial_multiple + ) + target_w = max( + spatial_multiple, round(width * factor / spatial_multiple) * spatial_multiple + ) + resized = functional.interpolate( + video.float(), + size=(target_h, target_w), + mode="bilinear", + align_corners=False, + ) + return resized.round_().clamp_(0, 255).to(torch.uint8) + + +def pad_to_spatial_factor( + video: torch.Tensor, spatial_factor: int +) -> tuple[torch.Tensor, tuple[int, int]]: + """Pad bottom/right to the VAE spatial factor; return the original size. + + Crop the scaled padding after SR to preserve the source framing.""" + if video.ndim != 4: + raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}") + height, width = video.shape[-2:] + pad_h = (-height) % spatial_factor + pad_w = (-width) % spatial_factor + if pad_h == 0 and pad_w == 0: + return video, (height, width) + padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate") + return padded, (height, width) + + +def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor: + """Crop [..., H, W] to hw from the top-left, undoing bottom/right padding.""" + height, width = hw + return video[..., :height, :width] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py new file mode 100644 index 0000000000..38bdb84299 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_io.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Streaming video / audio input and output helpers of Kandinsky 6 video SR. + +Decoding follows the reference CLI (``read_video_tchw_uint8`` -> +``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams: +only the frames that survive fps resampling and the 121-frame cap are converted to RGB +and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily +(it is part of the ``diffusion`` extra). +""" + +import msgspec +import numpy as np +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( + MAX_NUM_FRAMES, + RESAMPLE_FPS_TOLERANCE, + TARGET_FPS, + clip_to_aligned_frames, +) + +SOURCE_AUDIO_SAMPLE_RATE = 44100 +OUTPUT_FRAME_CHUNK = 8 + + +class DecodedClip(msgspec.Struct, frozen=True): + """Frames ready for SR plus the rate they play at.""" + + frames: torch.Tensor # [T, 3, H, W] uint8, T = 1 + 8k <= 121 + fps: int # effective fps (24 when downsampled, else round(source fps)) + source_fps: float + + +class FrameSelector: + """Streaming form of ``resample_to_target_fps`` + the first-121-frames cap. + + Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the + selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further + frame can change the result, and :meth:`limit` trims the selection once the source + length is known. For a source faster than the target the selected indices are + ``round(i * step)``, ``step = source_fps / target_fps``, of which there are + ``int(total / step)`` (those below ``total``), exactly as in the reference; + otherwise every frame is selected. + """ + + def __init__( + self, + source_fps: float, + *, + target_fps: int = TARGET_FPS, + max_frames: int = MAX_NUM_FRAMES, + ) -> None: + self.max_frames = max_frames + self.downsample = ( + abs(source_fps - target_fps) >= RESAMPLE_FPS_TOLERANCE + and source_fps > target_fps + ) + self.step = source_fps / target_fps + self.effective_fps = target_fps if self.downsample else round(source_fps) + self._kept = 0 + self._next_index = 0 + + def keep(self, index: int) -> bool: + if self._kept >= self.max_frames: + return False + return not self.downsample or index == self._next_index + + def mark_kept(self) -> None: + self._kept += 1 + if self.downsample: + self._next_index = round(self._kept * self.step) + + def enough(self, frames_seen: int) -> bool: + """True once ``max_frames`` are kept and the source is known to be long enough.""" + if self._kept < self.max_frames: + return False + return not self.downsample or int(frames_seen / self.step) >= self.max_frames + + def limit(self, total_frames: int, kept: int) -> int: + """Number of kept frames the reference would return for a source of this length.""" + if self.downsample: + kept = min(kept, int(total_frames / self.step)) + return min(kept, self.max_frames) + + +def decode_clip( + path: str, + *, + target_fps: int = TARGET_FPS, + max_frames: int = MAX_NUM_FRAMES, +) -> DecodedClip: + """Decode the first video stream into the aligned uint8 clip the SR model expects.""" + import av + + frames: list[torch.Tensor] = [] + seen = 0 + with av.open(str(path), mode="r") as container: + stream = container.streams.video[0] + source_fps = float(stream.average_rate or stream.base_rate or 0.0) + if source_fps <= 0: + raise ValueError(f"Video {path} reports no usable fps (got {source_fps}).") + selector = FrameSelector( + source_fps, target_fps=target_fps, max_frames=max_frames + ) + if selector.effective_fps < 1: + raise ValueError(f"Video {path} has an unusable fps of {source_fps}.") + stream.thread_type = "AUTO" + for frame in container.decode(stream): + if selector.keep(seen): + array = frame.to_ndarray(format="rgb24") + frames.append(torch.from_numpy(array).permute(2, 0, 1)) + selector.mark_kept() + seen += 1 + if selector.enough(seen): + break + kept = selector.limit(seen, len(frames)) + if kept == 0: + raise ValueError(f"Video {path} has no readable frames.") + video = torch.stack(frames[:kept]).contiguous() + return DecodedClip( + frames=clip_to_aligned_frames(video, max_frames), + fps=selector.effective_fps, + source_fps=source_fps, + ) + + +def extract_source_audio( + path: str, + *, + sample_rate: int = SOURCE_AUDIO_SAMPLE_RATE, + max_seconds: float | None = None, +) -> np.ndarray | None: + """Decode the first audio stream as mono float32 at ``sample_rate`` (None if absent). + + ``max_seconds`` stops decoding once that much audio has been produced; the result is + trimmed to exactly that length. + """ + import av + + if sample_rate <= 0: + raise ValueError("sample_rate must be positive") + limit = None if max_seconds is None else round(max_seconds * sample_rate) + chunks: list[np.ndarray] = [] + produced = 0 + with av.open(str(path), mode="r") as container: + if not container.streams.audio: + return None + resampler = av.AudioResampler(format="fltp", layout="mono", rate=sample_rate) + for frame in container.decode(audio=0): + for resampled in resampler.resample(frame): + chunk = resampled.to_ndarray().reshape(-1) + chunks.append(chunk) + produced += chunk.shape[0] + if limit is not None and produced >= limit: + break + else: + for resampled in resampler.resample(None): + chunks.append(resampled.to_ndarray().reshape(-1)) + if not chunks: + return None + audio = np.concatenate(chunks).astype(np.float32, copy=False) + return audio if limit is None else audio[:limit] + + +def synthetic_clip(*, height: int, width: int, frames: int) -> torch.Tensor: + """Deterministic ``[T, 3, H, W]`` uint8 clip (warmup input; no file needed).""" + shape = (frames, 1, height, width) + ys = torch.linspace(0, 255, height).view(1, 1, height, 1).expand(shape) + xs = torch.linspace(0, 255, width).view(1, 1, 1, width).expand(shape) + ts = torch.linspace(0, 255, frames).view(frames, 1, 1, 1).expand(shape) + return (torch.cat([ys, xs, ts], dim=1) / 2 + 64).clamp(0, 255).to(torch.uint8) + + +def to_output_video(video: torch.Tensor) -> torch.Tensor: + """uint8 ``[3, T, H, W]`` -> fp16 ``[1, 3, T, H, W]`` in [0, 1] that saves losslessly. + + The framework writers truncate ``(x * 255)`` to uint8. fp16 halves the memory of a + large clip, but a plain ``k / 255`` stored in fp16 truncates to ``k - 1`` for over half + of the values when a consumer upcasts before multiplying. ``(k + 0.5) / 255`` stays + inside the bucket of ``k`` in every evaluation order. Converted in frame chunks to + bound the float32 temporaries. + """ + channels, frames, height, width = video.shape + out = torch.empty((1, channels, frames, height, width), dtype=torch.float16) + for start in range(0, frames, OUTPUT_FRAME_CHUNK): + chunk = video[:, start : start + OUTPUT_FRAME_CHUNK] + out[0, :, start : start + OUTPUT_FRAME_CHUNK] = ( + (chunk.float() + 0.5) / 255.0 + ).to(torch.float16) + return out diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py new file mode 100644 index 0000000000..dc466b53fe --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/kandinsky6_sr/video_utils.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +# Vendored from sr_core (parity-tested against the k6_video reference); adapted from +# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0). +"""Frame selection and bounded-memory output resize for Kandinsky 6 SR.""" + +from __future__ import annotations + +import logging + +import torch +from torch.nn import functional as f # noqa: N812 + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import ( + ASPECT_MISMATCH_TOLERANCE, +) + +logger = logging.getLogger(__name__) + +# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) --- +# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames +# (= 1 + 8*15); feeding more frames or another rate is out of contract. +TARGET_FPS: int = 24 +MAX_NUM_FRAMES: int = 121 +# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no +# resample): covers 23.98 (NTSC) and 25.00 sources. +RESAMPLE_FPS_TOLERANCE: float = 1.5 +# Frames per interpolate call when downscaling the output (a 2K clip is +# ~2.4 GB as uint8, so the resize walks the clip in chunks). +RESIZE_FRAME_CHUNK: int = 8 + + +def select_frame_indices( + total_frames: int, src_fps: float, target_fps: float +) -> list[int]: + """Select rounded frame indices to downsample src_fps to target_fps.""" + step = src_fps / target_fps + indices = [round(i * step) for i in range(int(total_frames / step))] + return [i for i in indices if i < total_frames] + + +def resample_to_target_fps( + video: torch.Tensor, + src_fps: float, + target_fps: int = TARGET_FPS, +) -> tuple[torch.Tensor, int]: + """Downsample high-fps inputs; retain near-target and lower-fps inputs. + + Return [T, C, H, W] video and its effective playback rate.""" + if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE: + return video, round(src_fps) + if src_fps > target_fps: + indices = select_frame_indices(video.shape[0], src_fps, target_fps) + logger.warning( + "Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); " + "source temporal detail beyond %sfps is discarded.", + src_fps, + target_fps, + video.shape[0], + len(indices), + target_fps, + ) + return video[indices], target_fps + logger.warning( + "Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); " + "output is mildly out of distribution.", + src_fps, + target_fps, + ) + return video, round(src_fps) + + +def clip_to_aligned_frames( + video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES +) -> torch.Tensor: + """Cap the frame count, then floor-align to 1+8k; reject empty video.""" + if max_num_frames > 0: + video = video[:max_num_frames] + aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0 + if aligned == 0: + msg = "Video has no readable frames." + raise ValueError(msg) + return video[:aligned] + + +def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor: + """Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping.""" + _c, _t, height, width = video.shape + target_h, target_w = target_hw + if (height, width) == (target_h, target_w): + return video + if target_h > height or target_w > width: + msg = ( + f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would " + f"upscale. Pick a lower tier or a higher --resolution-scale for this source." + ) + raise ValueError(msg) + + source_ratio, target_ratio = width / height, target_w / target_h + if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE: + logger.warning( + "Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically " + "(no crop), the picture will be squeezed by %.1f%%.", + width, + height, + source_ratio, + target_w, + target_h, + target_ratio, + abs(source_ratio / target_ratio - 1) * 100, + ) + + channels, frames = video.shape[0], video.shape[1] + out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8) + for start in range(0, frames, RESIZE_FRAME_CHUNK): + chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float() + resized = f.interpolate( + chunk, + size=(target_h, target_w), + mode="bilinear", + antialias=True, + align_corners=False, + ) + out[:, start : start + RESIZE_FRAME_CHUNK] = ( + resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3) + ) + + logger.info( + "Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)", + width, + height, + target_w, + target_h, + width / target_w, + height / target_h, + ) + return out diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py index 98c62e3037..5807fbce2d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py @@ -12,7 +12,13 @@ import inspect import torch from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput -from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput +from sglang.multimodal_gen.configs.pipeline_configs.base import ( + PipelineConfig, + TextConditioningOutput, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) from sglang.multimodal_gen.runtime.cache.conditioning import ( cached_encoder_call, prefer_conditioning_cache, @@ -106,6 +112,26 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group): ) +def _text_encoder_max_length_is_fixed( + pipeline_config: PipelineConfig, encoder_index: int +) -> bool: + """True when ``encoder_index``'s tokenizer ``max_length`` is + architecturally fixed and must never be overridden by a request's + ``max_sequence_length``. + + Flux v1's encoder 0 and Kandinsky6 TI2VA's encoder 1 are both CLIP with + a fixed 77-token pooled-embedding context; overriding it corrupts CLIP + tokenization. Every other encoder (Flux v1's encoder 1 T5, Kandinsky6's + encoder 0 Reason1/Qwen, and every other pipeline's encoders) is meant to + take the request's override, so this returns False for them. + """ + if pipeline_config.is_flux_v1(): + return encoder_index == 0 + if isinstance(pipeline_config, Kandinsky6TI2VAPipelineConfig): + return encoder_index == 1 + return False + + def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor: base_shape = list(tensors[0].shape) for tensor in tensors[1:]: @@ -593,10 +619,12 @@ class TextEncodingStage(ConditionEncodingStage): encoder_config.tokenizer_kwargs, **text_encoder_extra_arg, ) - # Pass max_length to tokenizer if specified in the request. Flux v1 encoder 0 - # is CLIP with a fixed 77-token context; overriding breaks tokenization. - is_flux_v1 = server_args.pipeline_config.is_flux_v1() - if max_length is not None and not (is_flux_v1 and i == 0): + # Pass max_length to tokenizer if specified in the request, except + # for a text encoder whose context is architecturally fixed (see + # _text_encoder_max_length_is_fixed). + if max_length is not None and not _text_encoder_max_length_is_fixed( + server_args.pipeline_config, i + ): tok_kwargs["max_length"] = max_length text_inputs: dict = server_args.pipeline_config.tokenize_prompt( diff --git a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py index 7eee4260b1..fef29346f0 100644 --- a/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/hf_diffusers_utils.py @@ -386,7 +386,8 @@ def _verify_diffusers_model_complete(path: str) -> bool: component_keys = [ key for key, value in model_index.items() - if _is_diffusers_component_entry(value) + if not key.startswith("_") + and _is_diffusers_component_entry(value) and any(item is not None for item in value) ] if component_keys: @@ -561,6 +562,35 @@ def get_diffusers_component_config( lambda acc, path: acc | load_dict(path), config_file_paths, {} ) + if "_class_name" not in combined_config: + # Some checkpoints (e.g. real Kandinsky6 exports, which declare + # "uses_patched_diffusers": true) omit "_class_name" from a + # component's own config.json, relying on the parent + # model_index.json's [library, class_name] entry for that component + # subfolder instead. Fall back to that so component loaders (which + # unconditionally pop "_class_name") still resolve the right class. + model_index_path = os.path.join( + os.path.dirname(os.path.normpath(component_path)), "model_index.json" + ) + component_key = os.path.basename(os.path.normpath(component_path)) + if os.path.exists(model_index_path): + try: + with open(model_index_path) as f: + model_index = json.load(f) + entry = model_index.get(component_key) + class_name = ( + entry[-1] if isinstance(entry, (list, tuple)) and entry else entry + ) + if class_name: + combined_config["_class_name"] = str(class_name) + except Exception as exc: + logger.warning( + "Failed to backfill _class_name for %s from %s: %s", + component_path, + model_index_path, + exc, + ) + quant_config = combined_config.get("quantization_config") if quant_config is not None: combined_config["quantization_config"] = normalize_flat_modelopt_quant_config( @@ -736,7 +766,8 @@ def verify_model_config_and_directory(model_path: str) -> dict[str, Any]: component_keys = [ key for key, value in config.items() - if isinstance(value, (list, tuple)) + if not key.startswith("_") + and isinstance(value, (list, tuple)) and len(value) == 2 and all(isinstance(item, str) for item in value) ] diff --git a/python/sglang/multimodal_gen/test/server/test_server_kandinsky6.py b/python/sglang/multimodal_gen/test/server/test_server_kandinsky6.py new file mode 100644 index 0000000000..22689a70e2 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_kandinsky6.py @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in checkpoint test; SGLANG_KANDINSKY6_TEST_SERVER_ARGS selects topology.""" + +import io +import json +import os +import sys +import time +from pathlib import Path + +import av +import numpy as np +import pytest +import requests +import torch +from huggingface_hub import hf_hub_download +from PIL import Image, ImageDraw +from safetensors.torch import save_file + +from sglang.multimodal_gen.test.server.test_server_utils import ServerManager +from sglang.multimodal_gen.test.test_utils import ( + extract_audio_pcm_from_video_bytes, + get_dynamic_server_port, +) + +pytestmark = pytest.mark.skipif( + not os.environ.get("SGLANG_KANDINSKY6_TEST_MODEL"), + reason="set SGLANG_KANDINSKY6_TEST_MODEL for full-checkpoint HTTP tests", +) + + +@pytest.fixture(scope="module") +def server(): + manager = ServerManager( + model=os.environ["SGLANG_KANDINSKY6_TEST_MODEL"], + port=get_dynamic_server_port(), + extra_args=( + "--performance-mode speed --attention-backend fa --warmup-mode off " + + os.environ.get("SGLANG_KANDINSKY6_TEST_SERVER_ARGS", "--num-gpus 1") + ), + ) + context = manager.start() + try: + yield context + finally: + context.cleanup() + + +def _generate_video(server, *, image=None, size="384x256", prompt=None, **overrides): + url = f"http://127.0.0.1:{server.port}/v1/videos" + payload = { + "prompt": prompt or "A red ceramic teapot on a wooden table. Quiet jazz music.", + "size": size, + "num_frames": 17, + "fps": 24, + "num_inference_steps": 2, + "seed": 42, + } | overrides + with requests.Session() as client: + if image is None: + response = client.post(url, json=payload, timeout=120) + else: + response = client.post( + url, + data=payload, + files={"input_reference": ("reference.png", image, "image/png")}, + timeout=120, + ) + assert response.ok, response.text + job_url = f"{url}/{response.json()['id']}" + try: + deadline = time.monotonic() + 300 + while True: + response = client.get(job_url, timeout=30) + assert response.ok, response.text + job = response.json() + assert job["status"] not in {"failed", "cancelled", "deleted"}, job + if job["status"] == "completed": + break + assert time.monotonic() < deadline, (job, server.log_tail()) + time.sleep(0.5) + response = client.get(f"{job_url}/content", timeout=60) + assert response.ok, response.text + content = response.content + finally: + client.delete(job_url, timeout=30) + with av.open(io.BytesIO(content)) as video: + frames = np.stack( + [frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)] + ) + audio = extract_audio_pcm_from_video_bytes(content) + width, height = map(int, size.split("x")) + assert frames.shape == (17, height, width, 3) + assert frames.std() > 1, "degenerate video" + assert audio.size > 0 and np.isfinite(audio).all() + assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5, "silent audio" + return frames, audio + + +def test_repeated_requests_and_conditioning_isolation(server): + reference = Image.new("RGB", (384, 256), "white") + ImageDraw.Draw(reference).rectangle((96, 64, 288, 192), fill="red") + buffer = io.BytesIO() + reference.save(buffer, format="PNG") + + first = _generate_video(server) + repeated = _generate_video(server) + for actual, expected in zip(repeated, first, strict=True): + np.testing.assert_array_equal(actual, expected) + + conditioned = _generate_video(server, image=buffer.getvalue()) + conditioned_repeat = _generate_video(server, image=buffer.getvalue()) + for actual, expected in zip(conditioned_repeat, conditioned, strict=True): + np.testing.assert_array_equal(actual, expected) + assert not np.array_equal(conditioned[0], first[0]), ( + "image conditioning was ignored" + ) + + _generate_video(server, size="256x384", prompt="A blue glass vase. Rain sounds.") + restored = _generate_video(server) + for actual, expected in zip(restored, first, strict=True): + np.testing.assert_array_equal(actual, expected) + + +def test_request_cache_dit_refresh_and_opt_out(server): + baseline = _generate_video(server, num_inference_steps=6, enable_cache_dit=False) + no_skip = _generate_video( + server, + num_inference_steps=6, + enable_cache_dit=True, + cache_dit_params={"max_warmup_steps": 6}, + ) + for actual, expected in zip(no_skip, baseline, strict=True): + np.testing.assert_array_equal(actual, expected) + + cached_params = dict( + num_inference_steps=6, + enable_cache_dit=True, + cache_dit_params={ + "max_warmup_steps": 1, + "scm_compute_bins": [1, 1, 1], + "scm_cache_bins": [1, 1, 1], + "scm_policy": "static", + }, + ) + first = _generate_video(server, **cached_params) + assert any(not np.array_equal(a, b) for a, b in zip(first, baseline, strict=True)) + _generate_video( + server, + size="256x384", + prompt="A blue glass vase. Rain sounds.", + **cached_params, + ) + repeated = _generate_video(server, **cached_params) + for actual, expected in zip(repeated, first, strict=True): + np.testing.assert_array_equal(actual, expected) + restored = _generate_video(server, num_inference_steps=6, enable_cache_dit=False) + for actual, expected in zip(restored, baseline, strict=True): + np.testing.assert_array_equal(actual, expected) + + +@pytest.fixture(scope="module") +def lora_adapter(tmp_path_factory): + model = os.environ["SGLANG_KANDINSKY6_TEST_MODEL"] + config_path = Path(model) / "transformer" / "config.json" + if not config_path.is_file(): + config_path = Path(hf_hub_download(model, "transformer/config.json")) + config = json.loads(config_path.read_text()) + layers = { + "visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj": ( + config["ff_dim"], + config["model_dim"], + ), + "visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2": ( + config["model_dim_a"], + config["ff_dim_a"], + ), + "video_time_embeddings.timestep_embedder.linear_1": ( + config["time_dim"], + config["model_dim"], + ), + } + generator = torch.Generator().manual_seed(827) + weights = {} + for name, (out_dim, in_dim) in layers.items(): + for suffix, shape in (("A", (2, in_dim)), ("B", (out_dim, 2))): + weights[f"transformer.{name}.lora_{suffix}.weight"] = ( + torch.randn(shape, generator=generator) * 0.05 + ) + path = tmp_path_factory.mktemp("kandinsky6_lora") / "adapter.safetensors" + save_file(weights, path) + return str(path) + + +@pytest.mark.parametrize("merge_mode", ["dynamic", "merge"]) +def test_lora_scaling_repeatability_and_restore(server, lora_adapter, merge_mode): + base_url = f"http://127.0.0.1:{server.port}/v1" + baseline = _generate_video(server, enable_cache_dit=False) + + def set_strength(strength): + response = requests.post( + f"{base_url}/set_lora", + json={ + "lora_nickname": "kandinsky6_synthetic", + "lora_path": lora_adapter, + "target": "transformer", + "strength": strength, + "merge_mode": merge_mode, + }, + timeout=180, + ) + assert response.ok, response.text + + try: + set_strength(1.0) + adapted = _generate_video(server, enable_cache_dit=False) + for actual, expected in zip(adapted, baseline, strict=True): + assert not np.array_equal(actual, expected), ( + "LoRA did not change the output" + ) + repeated = _generate_video(server, enable_cache_dit=False) + for actual, expected in zip(repeated, adapted, strict=True): + np.testing.assert_array_equal(actual, expected) + set_strength(0.0) + zero = _generate_video(server, enable_cache_dit=False) + for actual, expected in zip(zero, baseline, strict=True): + np.testing.assert_array_equal(actual, expected) + set_strength(0.5) + scaled = _generate_video(server, enable_cache_dit=False) + assert any( + not np.array_equal(a, b) for a, b in zip(scaled, adapted, strict=True) + ) + set_strength(1.0) + restored_adapter = _generate_video(server, enable_cache_dit=False) + for actual, expected in zip(restored_adapter, adapted, strict=True): + np.testing.assert_array_equal(actual, expected) + finally: + response = requests.post( + f"{base_url}/unmerge_lora_weights", + json={"target": "transformer"}, + timeout=180, + ) + assert response.ok, response.text + restored_base = _generate_video(server, enable_cache_dit=False) + for actual, expected in zip(restored_base, baseline, strict=True): + np.testing.assert_array_equal(actual, expected) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/python/sglang/multimodal_gen/test/server/test_server_kandinsky6_sr.py b/python/sglang/multimodal_gen/test/server/test_server_kandinsky6_sr.py new file mode 100644 index 0000000000..2932b5a537 --- /dev/null +++ b/python/sglang/multimodal_gen/test/server/test_server_kandinsky6_sr.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in SR checkpoint test covering real JSON and multipart video requests.""" + +import io +import os +import shlex +import subprocess +import time + +import av +import numpy as np +import pytest +import requests + +from sglang.multimodal_gen.test.server.test_server_utils import ServerManager +from sglang.multimodal_gen.test.test_utils import ( + extract_audio_pcm_from_video_bytes, + get_dynamic_server_port, +) + +pytestmark = pytest.mark.skipif( + not os.environ.get("SGLANG_KANDINSKY6_SR_TEST_MODEL"), + reason="set SGLANG_KANDINSKY6_SR_TEST_MODEL for full-checkpoint HTTP tests", +) + + +@pytest.fixture(scope="module") +def source_videos(tmp_path_factory): + directory = tmp_path_factory.mktemp("kandinsky6_sr_source") + source = directory / "source.mp4" + silent = directory / "silent.mp4" + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=384x256:rate=24", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=44100", + "-frames:v", + "33", + "-t", + str(33 / 24), + "-c:v", + "libx264", + "-crf", + "18", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + str(source), + ], + check=True, + timeout=60, + ) + subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(source), + "-c:v", + "copy", + "-an", + str(silent), + ], + check=True, + timeout=60, + ) + return source, silent + + +@pytest.fixture(scope="module") +def server(tmp_path_factory): + directory = tmp_path_factory.mktemp("kandinsky6_sr_server") + manager = ServerManager( + model=os.environ["SGLANG_KANDINSKY6_SR_TEST_MODEL"], + port=get_dynamic_server_port(), + extra_args=( + "--performance-mode speed --attention-backend fa --warmup-mode off " + f"--input-save-path {shlex.quote(str(directory / 'uploads'))} " + f"--output-path {shlex.quote(str(directory / 'outputs'))} " + + os.environ.get("SGLANG_KANDINSKY6_SR_TEST_SERVER_ARGS", "--num-gpus 1") + ), + ) + context = manager.start() + try: + yield context + finally: + context.cleanup() + + +def _generate( + server, source, *, scale=2, tiles_batch_size=1, upload=False, has_audio=True +): + url = f"http://127.0.0.1:{server.port}/v1/videos" + payload = { + "sr_resolution_scale": scale, + "sr_tiles_batch_size": tiles_batch_size, + "num_inference_steps": 2, + "seed": 42, + } + with requests.Session() as client: + if upload: + with source.open("rb") as stream: + response = client.post( + url, + data=payload, + files={"video_reference": (source.name, stream, "video/mp4")}, + timeout=120, + ) + else: + response = client.post( + url, json=payload | {"video_path": str(source)}, timeout=120 + ) + assert response.ok, response.text + job_url = f"{url}/{response.json()['id']}" + try: + deadline = time.monotonic() + 300 + while True: + response = client.get(job_url, timeout=30) + assert response.ok, response.text + job = response.json() + assert job["status"] not in {"failed", "cancelled", "deleted"}, job + if job["status"] == "completed": + break + assert time.monotonic() < deadline, (job, server.log_tail()) + time.sleep(0.5) + response = client.get(f"{job_url}/content", timeout=60) + assert response.ok, response.text + content = response.content + finally: + deleted = client.delete(job_url, timeout=30) + assert deleted.ok, deleted.text + with av.open(io.BytesIO(content)) as video: + assert bool(video.streams.audio) == has_audio + assert float(video.streams.video[0].average_rate) == 24 + frames = np.stack( + [frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)] + ) + assert frames.shape == (33, int(256 * scale), int(384 * scale), 3) + assert frames.std() > 1, "degenerate video" + audio = extract_audio_pcm_from_video_bytes(content) if has_audio else np.empty(0) + assert np.isfinite(audio).all() + if has_audio: + assert audio.size > 0 + assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5 + return frames, audio + + +def test_sr_requests_preserve_video_and_audio(server, source_videos): + source, silent = source_videos + first = _generate(server, source) + uploaded = _generate(server, source, upload=True) + for actual, expected in zip(uploaded, first, strict=True): + np.testing.assert_array_equal(actual, expected) + _generate(server, source, scale=4, upload=True) + _generate(server, source, scale=2.25, tiles_batch_size=2) + silent_output = _generate(server, silent, upload=True, has_audio=False) + np.testing.assert_array_equal(silent_output[0], first[0]) + restored = _generate(server, source) + for actual, expected in zip(restored, first, strict=True): + np.testing.assert_array_equal(actual, expected) diff --git a/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_release_configs.py b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_release_configs.py new file mode 100644 index 0000000000..a350e152e6 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_release_configs.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The real component configs (no weights) of the two official Kandinsky 6 VSR Diffusers repos, verbatim from the Hub. + +* flow-matching ``kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers``: ``FlowMatchEulerDiscreteScheduler``, a 64-wide DiT head; +* 2-step pi-Flow ``kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers``: ``PiflowScheduler`` (``nfe`` 2, + ``n_grid`` 10), a 640-wide DiT head. + +Both repos have the same DiT tensor names (only ``out_layer`` differs in width) and byte-identical ``vae/`` and +``latent_upscaler/`` files. +""" + +import json + +# --- kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers (flow-matching) --- +# The dict-valued ``_kandinsky6_sr`` entry of its model_index.json and the root ``sr_config.json`` exist only in this repo. +FLOW_MODEL_INDEX = json.loads( + r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.39.0","_kandinsky6_sr":{"format_version":1,"latent_upscaler":"latent_upscaler","portable_components":true,"source_checkpoint":"9454578e88f86dd9c40df6a8a79529a7e5a5ecc9","uses_patched_diffusers":true,"vae_name":"video-kvae"},"latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","FlowMatchEulerDiscreteScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}""" +) +FLOW_SR_CONFIG = json.loads( + r"""{"component_loading":"Diffusers ModelMixin wrappers","default_resolution_scale":2.25,"description":"Kandinsky 6 SR Diffusers bundle"}""" +) +FLOW_TRANSFORMER_CONFIG = json.loads( + r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":200,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":64,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":5.0,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}""" +) +FLOW_SCHEDULER_CONFIG = json.loads( + r"""{"_class_name":"FlowMatchEulerDiscreteScheduler","_diffusers_version":"0.39.0","base_image_seq_len":256,"base_shift":0.5,"invert_sigmas":false,"max_image_seq_len":4096,"max_shift":1.15,"num_train_timesteps":1000,"shift":5.0,"shift_terminal":null,"stochastic_sampling":false,"time_shift_type":"exponential","use_beta_sigmas":false,"use_dynamic_shifting":false,"use_exponential_sigmas":false,"use_karras_sigmas":false}""" +) + +# --- kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers (2-step pi-Flow) --- +DISTILLED_MODEL_INDEX = json.loads( + r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.41.0.dev0","latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","PiflowScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}""" +) +DISTILLED_TRANSFORMER_CONFIG = json.loads( + r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":0,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":640,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":3.5,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}""" +) +DISTILLED_SCHEDULER_CONFIG = json.loads( + r"""{"_class_name":"PiflowScheduler","_diffusers_version":"0.41.0.dev0","eps":1e-06,"final_step_size_scale":0.5,"n_grid":10,"nfe":2,"num_policy_substeps":128,"num_train_timesteps":1000,"shift":3.5}""" +) + +# --- identical in both repos (the VAE config stores the *string* "None" for unused channel fields) --- +VAE_CONFIG = json.loads( + r"""{"decoder_config":{"ch":256,"ch_mult":[0.0625,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"in_channels":"None","norm_type":"rms_norm","num_res_blocks":2,"out_ch":3,"padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"encoder_config":{"ch":128,"ch_mult":[0.125,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"double_z":true,"downsample_version":2,"fix_pxs":true,"in_channels":3,"norm_type":"rms_norm","num_res_blocks":2,"out_ch":"None","padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"scaling_factor":0.910344004631042,"spatial_factor":16,"temporal_factor":4,"vae_type":"video-kvae"}""" +) +LATENT_UPSCALER_CONFIG = json.loads( + r"""{"models":[{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":false,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4},"target_scale":"4x"},{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":true,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4,"x2_adapter_blocks":2,"x2_adapter_sources":[0,4],"x2_finisher":"pxs_residual","x2_tail_mode":"private_full"},"target_scale":"2x"}],"scaling_factor":0.910344004631042}""" +) diff --git a/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py new file mode 100644 index 0000000000..70167e2fa0 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/kandinsky6_sr_tiny_components.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tiny random Kandinsky 6 SR components and requests for the stage-level tests.""" + +import copy +from collections.abc import Callable, Mapping, Sequence +from types import SimpleNamespace +from typing import Any + +import torch + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( + Kandinsky6SRTransformer3DModel, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_REQUESTED_HW_KEY, + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + DitSpec, + SamplingSpec, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + LatentUpscaleFn, + decode_chunks, + denoise_chunks, + encode_video_to_lr_latent, + plan_tiles, + prepare_lu_tile_latents, + prepare_pixel_tile_latents, + stitch_tiles, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + RESOLUTIONS, + VAE_SPATIAL_FACTOR, +) + +# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels. +TINY_DIT = dict( + in_visual_dim=4, + in_text_dim=8, + in_text_dim2=8, + time_dim=16, + out_visual_dim=4, + patch_size=[1, 2, 2], + model_dim=32, + ff_dim=64, + num_text_blocks=0, + num_visual_blocks=2, + axes_dims=[8, 4, 4], + visual_cond=False, + instruct_type="noise", + use_text=False, + n_grid=3, + piflow_nfe=2, + piflow_num_policy_substeps=8, + piflow_final_step_size_scale=0.5, + piflow_shift=5.0, + piflow_eps=1e-6, + attribute_overrides={"instruct_type": "noise"}, + sr_visual_size=[512], + sr_scale_factor={"512": [1.0, 1.0, 1.0]}, +) +TINY_KVAE = dict( + ch=8, + ch_mult=(1, 1, 2, 2, 2), + num_res_blocks=2, + z_channels=4, + temporal_compress_times=4, + norm_type="rms_norm", +) +TINY_LU_MODEL = { + "architecture": "multi_scale", + "in_channels": 4, + "hidden_channels": 8, + "num_pre_blocks": 1, + "num_mid_blocks": 2, + "num_post_blocks": 2, + "expand_ratio": 2, + "dims": 3, + "bare_stem": True, + "input_skip": False, + "modulated_norm": True, + "modulated_output_proj": True, + "temporal_padding": "replicate", + "upsample_mode": "pxs_v2", + "upsample_padding_mode": "zeros", + "enable_x2_entry": True, + "x2_adapter_blocks": 1, + "x2_tail_mode": "private_full", + "x2_finisher": "pxs_residual", +} + + +def _randomize(module, seed): + generator = torch.Generator().manual_seed(seed) + with torch.no_grad(): + for param in module.parameters(): + param.copy_(torch.randn(param.shape, generator=generator) * 0.05) + + +def build_components(bank_scales, dit_arch=TINY_DIT): + """``(vae, dit, latent-upscaler bank or None, server args)`` with random weights.""" + vae_config = Kandinsky6SRVAEConfig() + vae_config.update_model_arch( + dict( + vae_type="video-kvae", + encoder_config=dict(TINY_KVAE, in_channels=3), + decoder_config=dict(TINY_KVAE, out_ch=3), + scaling_factor=0.5, + ) + ) + vae = Kandinsky6SRVAE(vae_config).eval() + _randomize(vae, 1) + dit_config = Kandinsky6SRDitConfig() + dit_config.update_model_arch(dict(dit_arch)) + dit = Kandinsky6SRTransformer3DModel(dit_config, dict(dit_arch)).eval() + _randomize(dit, 2) + bank = None + if bank_scales: + models = [ + {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} + for scale in bank_scales + ] + bank = Kandinsky6SRLatentUpscalerBank( + models, + 0.5, + scales=tuple(int(scale.removesuffix("x")) for scale in bank_scales), + ).eval() + _randomize(bank, 3) + # keep initialization reproducible on CPU, then exercise the native device kernels + device = get_local_torch_device() + vae.to(device) + dit.to(device) + if bank is not None: + bank.to(device) + pipeline_config = Kandinsky6SRPipelineConfig() + pipeline_config.dit_config = dit_config + server_args = SimpleNamespace( + pipeline_config=pipeline_config, + component_precisions={}, + # Kandinsky6SRDenoisingStage reuses the shared DenoisingStage's cache-dit / + # torch.compile wiring (_maybe_enable_cache_dit_and_torch_compile), which reads + # these; both stay off for this fixture. + enable_breakable_cuda_graph=False, + enable_torch_compile=False, + ) + return vae, dit, bank, server_args + + +def make_stage(stage_cls, server_args, *args, **kwargs): + stage = stage_cls(*args, **kwargs) + stage.server_args = server_args # component_uses() reads the bf16 precisions + return stage + + +def make_request(video, *, tiles_batch_size=1, num_steps=5): + params = Kandinsky6SRSamplingParams( + video_path="unused.mp4", + sr_resolution_scale=2, + sr_tiles_batch_size=tiles_batch_size, + num_inference_steps=num_steps, + seed=42, + ) + batch = Req(sampling_params=params) + batch.extra[SR_VIDEO_KEY] = video.clone() + batch.extra[SR_TILING_SCALE_KEY] = 2 + batch.extra[SR_REQUESTED_HW_KEY] = (video.shape[-2], video.shape[-1]) + return batch + + +def random_video(frames, height, width, seed=0): + generator = torch.Generator().manual_seed(seed) + shape = (frames, 3, height, width) + return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator) + + +@torch.no_grad() +def super_resolve( + video: torch.Tensor, + *, + vae: torch.nn.Module, + scaling_factor: float, + dit: Callable[..., torch.Tensor], + dit_spec: DitSpec, + spec: SamplingSpec, + scheduler: Any, + device: torch.device | str, + upscale_fn: LatentUpscaleFn | None = None, + lu_dtype: torch.dtype | None = None, + spatial_factor: int = VAE_SPATIAL_FACTOR, + resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS, +) -> torch.Tensor: + """Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``. + + ``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the + already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded + component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits. + """ + device = torch.device(device) + common = dict( + visual_size=spec.visual_size, + tiling_scale=spec.tiling_scale, + tile_min_overlap=spec.tile_min_overlap, + spatial_factor=spatial_factor, + resolutions=resolutions, + ) + if upscale_fn is not None: + lr_latent = encode_video_to_lr_latent(video, vae, device=device).cpu() + frame_hw = ( + lr_latent.shape[-2] * spatial_factor, + lr_latent.shape[-1] * spatial_factor, + ) + plan = plan_tiles(frame_hw=frame_hw, **common) + chunks = prepare_lu_tile_latents( + lr_latent, + plan, + upscale_fn=upscale_fn, + lu_dtype=lu_dtype, + scaling_factor=scaling_factor, + dit_spec=dit_spec, + spec=spec, + device=device, + spatial_factor=spatial_factor, + ) + else: + plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common) + chunks = prepare_pixel_tile_latents( + video, + plan, + vae=vae, + scaling_factor=scaling_factor, + dit_spec=dit_spec, + spec=spec, + device=device, + ) + denoised = denoise_chunks( + chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device + ) + tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device) + return stitch_tiles(tiles, plan) diff --git a/python/sglang/multimodal_gen/test/unit/test_cache_dit_integration.py b/python/sglang/multimodal_gen/test/unit/test_cache_dit_integration.py index b364fdf663..3dd87c944c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cache_dit_integration.py +++ b/python/sglang/multimodal_gen/test/unit/test_cache_dit_integration.py @@ -5,7 +5,7 @@ import sys import types import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, patch class _FakeDBCacheConfig: @@ -19,6 +19,7 @@ class _FakeDBCacheConfig: class _FakeForwardPattern: # A class (not a SimpleNamespace instance) so it is a valid type in # annotations like List[ForwardPattern], matching the real Enum. + Pattern_0 = "Pattern_0" Pattern_2 = "Pattern_2" Pattern_3 = "Pattern_3" @@ -155,8 +156,10 @@ def _install_torch_stub(): torch_nn.Module = _FakeModule torch_dist.ProcessGroup = _FakeProcessGroup torch_dist.ReduceOp = _FakeReduceOp + torch_dist.is_initialized = lambda: False torch.distributed = torch_dist torch.nn = torch_nn + torch.Tensor = object return { "torch": torch, @@ -218,7 +221,50 @@ def _import_module_with_legacy_stub(): return module, stub_modules +class TestCacheDitSimilarityGroups(unittest.TestCase): + def test_independent_cfg_or_dp_peers_do_not_reduce_over_world(self): + for group in (None, "singleton-sp"): + with self.subTest(group=group): + module = _import_module_with_stub() + original = Mock(return_value=True) + manager_type = type( + "CachedContextManager", (), {"similarity": original} + ) + contexts = types.ModuleType("cache_dit.caching.cache_contexts") + contexts.cache_manager = types.SimpleNamespace( + CachedContextManager=manager_type + ) + module.dist.get_world_size = Mock(return_value=1) + with patch.dict(sys.modules, {contexts.__name__: contexts}): + module._patch_cache_dit_similarity() + manager = manager_type() + manager._sglang_sp_group = group + manager.similarity( + "previous", "current", threshold=0.1, parallelized=True + ) + self.assertFalse(original.call_args.kwargs["parallelized"]) + + class TestCacheDitRefreshContext(unittest.TestCase): + def test_refresh_preserves_explicit_scm_mask_and_policy(self): + module = _import_module_with_stub() + module.refresh_context_on_transformer( + "transformer", + 4, + scm_preset="fast", + steps_computation_mask=[1, 0, 1, 0], + steps_computation_policy="static", + ) + self.assertEqual(module.cache_dit.steps_mask_calls, []) + self.assertEqual( + module.cache_dit.refresh_calls[0]["cache_config"], + { + "num_inference_steps": 4, + "steps_computation_mask": [1, 0, 1, 0], + "steps_computation_policy": "static", + }, + ) + def test_refresh_context_without_scm_preset_skips_steps_mask(self): module = _import_module_with_stub() module.refresh_context_on_transformer( @@ -298,6 +344,51 @@ def _make_transformer(class_name, layers=None): class TestBuildCustomBlockAdapter(unittest.TestCase): + def test_fsdp_subclass_preserves_native_adapter(self): + module = _import_module_with_stub() + for name, attr, multimodal, pattern in ( + ( + "Kandinsky6Transformer3DModel", + "visual_transformer_blocks", + True, + "Pattern_0", + ), + ( + "Kandinsky6Transformer3DModel", + "visual_transformer_blocks", + False, + "Pattern_2", + ), + ("AnimaTransformer3DModel", "transformer_blocks", False, "Pattern_3"), + ): + with self.subTest(name=name, multimodal=multimodal): + blocks = ["block_0", "block_1"] + native = type(name, (), {attr: blocks}) + fsdp_mixin = type("FSDPModule", (), {}) + transformer = type(f"FSDP{name}", (fsdp_mixin, native), {})() + transformer.config = types.SimpleNamespace(is_multimodal=multimodal) + adapter = module._build_custom_block_adapter( + transformer, has_separate_cfg=True + ) + self.assertIs(adapter.transformer, transformer) + self.assertIs(adapter.blocks, blocks) + self.assertEqual(adapter.forward_pattern, pattern) + self.assertTrue(adapter.has_separate_cfg) + + def test_kandinsky6_caches_both_joint_streams(self): + module = _import_module_with_stub() + transformer = _make_transformer("Kandinsky6Transformer3DModel") + transformer.visual_transformer_blocks = ["joint_block"] + for multimodal, pattern in ((True, "Pattern_0"), (False, "Pattern_2")): + with self.subTest(multimodal=multimodal): + transformer.config = types.SimpleNamespace(is_multimodal=multimodal) + adapter = module._build_custom_block_adapter( + transformer, has_separate_cfg=True + ) + self.assertIs(adapter.blocks, transformer.visual_transformer_blocks) + self.assertEqual(adapter.forward_pattern, pattern) + self.assertTrue(adapter.has_separate_cfg) + def test_builds_adapter_for_registered_class(self): module = _import_module_with_stub() blocks = ["block_0", "block_1"] diff --git a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py index 7eb9c68d98..bde9d6513a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py +++ b/python/sglang/multimodal_gen/test/unit/test_cache_dit_per_request.py @@ -99,10 +99,11 @@ def _batch( enable_cache_dit=None, cache_dit_params=None, is_warmup=False, + do_classifier_free_guidance=False, ) -> SimpleNamespace: return SimpleNamespace( is_warmup=is_warmup, - do_classifier_free_guidance=False, + do_classifier_free_guidance=do_classifier_free_guidance, sampling_params=SimpleNamespace( enable_cache_dit=enable_cache_dit, cache_dit_params=cache_dit_params, @@ -116,6 +117,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase): self.stage.server_args = SimpleNamespace( enable_breakable_cuda_graph=False, enable_torch_compile=False, + enable_cfg_parallel=False, ) self.stage.transformer = object() self.stage.transformer_2 = None @@ -127,9 +129,12 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase): self.enable_calls = [] self.disable_calls = [] self.refresh_calls = [] + self.cfg_modes = [] + self.refresh_options = [] def fake_enable(transformer, config, **kwargs): self.enable_calls.append(config) + self.cfg_modes.append(kwargs["has_separate_cfg"]) return transformer def fake_disable(transformer): @@ -138,6 +143,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase): def fake_refresh(transformer, num_inference_steps, scm_preset=None, **kwargs): self.refresh_calls.append(num_inference_steps) + self.refresh_options.append(kwargs) patchers = [ patch.object(denoising_module, "enable_cache_on_transformer", fake_enable), @@ -157,7 +163,42 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase): self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True)) self.assertEqual(len(self.enable_calls), 1) self.assertTrue(self.stage._cache_dit_enabled) - self.assertEqual(self.stage._cache_dit_active_key, cache_dit_overrides_key({})) + self.assertEqual( + self.stage._cache_dit_active_key, (cache_dit_overrides_key({}), False) + ) + + def test_cfg_change_remounts_and_parallel_cfg_has_one_branch_per_rank(self): + self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True)) + cfg_batch = _batch(enable_cache_dit=True, do_classifier_free_guidance=True) + self.stage._maybe_enable_cache_dit(8, cfg_batch) + self.stage.server_args.enable_cfg_parallel = True + self.stage._maybe_enable_cache_dit(8, cfg_batch) + self.assertEqual(self.cfg_modes, [False, True, False]) + self.assertEqual(len(self.disable_calls), 2) + + def test_repeated_custom_scm_request_keeps_mask_and_policy(self): + batch = _batch( + enable_cache_dit=True, + cache_dit_params={ + "scm_compute_bins": [2, 2], + "scm_cache_bins": [2, 2], + "scm_policy": "static", + }, + ) + self.stage._maybe_enable_cache_dit(8, batch) + self.stage._maybe_enable_cache_dit(8, batch) + self.assertEqual(len(self.enable_calls), 1) + self.assertEqual( + self.refresh_options, + [ + { + "steps_computation_mask": self.enable_calls[ + 0 + ].steps_computation_mask, + "steps_computation_policy": "static", + } + ], + ) def test_server_default_off_unmounts_after_request_enable(self): self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True)) diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan_vae_frame_attention.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan_vae_frame_attention.py new file mode 100644 index 0000000000..1b86fed31d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan_vae_frame_attention.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 + +import weakref +from unittest.mock import patch + +import pytest +import torch +from torch.nn.attention import SDPBackend, sdpa_kernel + +from sglang.multimodal_gen.runtime.layers.parallel_conv import ( + disable_spatial_parallel_decode, +) +from sglang.multimodal_gen.runtime.models.vaes.hunyuanvae import ( + HunyuanVAEAttention, + HunyuanVideoMidBlock3D, + HunyuanVideoUpsampleCausal3D, + prepare_causal_attention_mask, +) + + +@pytest.mark.parametrize("frames", [1, 3, 7]) +@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16]) +def test_frame_attention_matches_dense_causal_mask(frames, batch_size, heads, dtype): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for the production SDPA path") + torch.manual_seed(0) + attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().to(dtype) + hidden = torch.randn(batch_size, frames * 15, 32, device="cuda", dtype=dtype) + mask = prepare_causal_attention_mask(frames, 15, dtype, hidden.device) + with torch.inference_mode(): + expected = attention(hidden, attention_mask=mask) + actual = attention(hidden, num_frames=frames) + # unmasked prefixes may select a different fused SDPA reduction kernel + tolerance = max(1e-6, torch.finfo(dtype).eps) + torch.testing.assert_close(actual, expected, rtol=tolerance, atol=tolerance) + + +@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)]) +def test_bf16_frame_attention_matches_dense_with_same_math_backend(batch_size, heads): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for the production SDPA path") + torch.manual_seed(0) + attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().bfloat16() + hidden = torch.randn(batch_size, 105, 32, device="cuda", dtype=torch.bfloat16) + mask = prepare_causal_attention_mask(7, 15, hidden.dtype, hidden.device) + with torch.inference_mode(), sdpa_kernel(SDPBackend.MATH): + expected = attention(hidden, attention_mask=mask) + actual = attention(hidden, num_frames=7) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_spatial_attention_avoids_dense_mask_and_preserves_height_shards(): + torch.manual_seed(0) + attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True) + stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D) + torch.nn.Module.__init__(stage) + stage.spatial_parallel = True + hidden = torch.randn(1, 8, 3, 5, 3) + local = hidden[:, :, :, :2] + mask = prepare_causal_attention_mask(3, 15, hidden.dtype, hidden.device) + with torch.inference_mode(): + expected = attention(hidden.permute(0, 2, 3, 4, 1).flatten(1, 3), mask) + expected = expected.unflatten(1, (3, 5, 3)).permute(0, 4, 1, 2, 3) + with ( + patch( + "sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.gather_variable_height", + return_value=(hidden, [2, 3]), + ) as gather, + patch( + "sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.chunk_height_by_sizes", + side_effect=lambda value, sizes: value[:, :, :, : sizes[0]], + ) as chunk, + patch( + "sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.prepare_causal_attention_mask", + side_effect=AssertionError( + "spatial decode must not build a dense mask" + ), + ), + ): + actual = stage._run_attention(attention, local) + gather.assert_called_once_with(local) + assert chunk.call_args.args[1] == [2, 3] + torch.testing.assert_close(actual, expected[:, :, :, :2]) + + +def test_frame_attention_preserves_gradient_flow(): + torch.manual_seed(0) + attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True).double() + hidden = torch.randn(1, 12, 8, dtype=torch.float64, requires_grad=True) + mask = prepare_causal_attention_mask(3, 4, hidden.dtype, hidden.device) + expected = attention(hidden, attention_mask=mask) + expected_grad = torch.autograd.grad(expected.square().sum(), hidden)[0] + actual = attention(hidden, num_frames=3) + actual_grad = torch.autograd.grad(actual.square().sum(), hidden)[0] + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(actual_grad, expected_grad) + + +def test_disabled_spatial_decode_preserves_dense_attention(): + torch.manual_seed(0) + attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True) + stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D) + torch.nn.Module.__init__(stage) + hidden = torch.randn(1, 8, 3, 5, 3) + with torch.inference_mode(): + stage.spatial_parallel = False + expected = stage._run_attention(attention, hidden) + stage.spatial_parallel = True + with ( + disable_spatial_parallel_decode(), + patch.object(attention, "forward", wraps=attention.forward) as forward, + ): + actual = stage._run_attention(attention, hidden) + assert "num_frames" not in forward.call_args.kwargs + assert forward.call_args.kwargs["attention_mask"].shape == (1, 45, 45) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_upsample_releases_intermediates_before_convolution(): + upsample = HunyuanVideoUpsampleCausal3D(4) + hidden = torch.randn(1, 4, 3, 5, 7) + interpolate = torch.nn.functional.interpolate + buffers = [] + + def track_interpolate(*args, **kwargs): + result = interpolate(*args, **kwargs) + buffers.append(weakref.ref(result)) + return result + + def check_input(value): + assert all(buffer() is None for buffer in buffers) + return value + + with ( + torch.inference_mode(), + patch( + "sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.F.interpolate", + side_effect=track_interpolate, + ), + patch.object(upsample.conv, "forward", side_effect=check_input), + ): + actual = upsample(hidden) + expected = torch.cat( + ( + interpolate(hidden[:, :, 0], scale_factor=2, mode="nearest").unsqueeze(2), + interpolate(hidden[:, :, 1:].contiguous(), scale_factor=2, mode="nearest"), + ), + dim=2, + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) diff --git a/python/sglang/multimodal_gen/test/unit/test_ipc_a2a_lifecycle.py b/python/sglang/multimodal_gen/test/unit/test_ipc_a2a_lifecycle.py index c6ed8ddc97..520af21155 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ipc_a2a_lifecycle.py +++ b/python/sglang/multimodal_gen/test/unit/test_ipc_a2a_lifecycle.py @@ -1,14 +1,51 @@ from unittest.mock import patch +import pytest + from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import ( IpcA2AState, _peer_cuda_device, + _Unsupported, ipc_a2a_ready, ) _IPC = "sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a" +@pytest.mark.parametrize("rank", [0, 1]) +@pytest.mark.parametrize("nvlink", [False, True]) +def test_ipc_requires_nvlink_before_mapping_peer_memory(monkeypatch, rank, nvlink): + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,1") + state = IpcA2AState() + with ( + patch(f"{_IPC}.dist.get_rank", return_value=rank), + patch(f"{_IPC}.torch.cuda.current_device", return_value=rank), + patch(f"{_IPC}._peer_cuda_device", return_value=1 - rank), + patch( + "sglang.multimodal_gen.runtime.platforms.current_platform.is_full_nvlink", + return_value=nvlink, + ) as topology, + patch(f"{_IPC}.torch.cuda.can_device_access_peer", return_value=True) as peer, + patch("ctypes.CDLL") as cudart, + patch(f"{_IPC}.load_ipc_a2a_sync") as kernels, + patch(f"{_IPC}.torch.zeros"), + patch.object(state, "_share"), + ): + if nvlink: + state.init(object()) + assert state.inited + peer.assert_called_once_with(rank, 1 - rank) + kernels.assert_called_once() + else: + with pytest.raises(_Unsupported, match="NVLink"): + state.init(object()) + assert not state.inited + peer.assert_not_called() + cudart.assert_not_called() + kernels.assert_not_called() + topology.assert_called_once_with([1, 3]) + + def test_reinitializes_ipc_transport_for_replaced_process_group(): state = IpcA2AState() old_group = object() diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_arch_config.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_arch_config.py new file mode 100644 index 0000000000..aa1f7eebe7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_arch_config.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint architecture validation and native parameter-name mapping.""" + +import pytest + +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6ArchConfig, + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAEConfig, +) +from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping + + +@pytest.mark.parametrize( + "overrides,expected", + [ + ({}, (48, 32, 128, (8, 8, 8))), + ({"ff_dim_a": 96, "axes_dims_a": (4, 4, 4)}, (48, 32, 96, (4, 4, 4))), + ({"time_dim_a": 16, "fix_modulation": True}, (48, 16, 128, (8, 8, 8))), + ({"time_dim_a": 16, "is_multimodal": False}, (48, 16, 128, (8, 8, 8))), + ], +) +def test_audio_dimension_inheritance_and_overrides(overrides, expected): + arch = Kandinsky6ArchConfig( + model_dim=48, time_dim=32, ff_dim=128, axes_dims=(8, 8, 8), **overrides + ) + assert ( + arch.model_dim_a, + arch.time_dim_a, + arch.ff_dim_a, + arch.axes_dims_a, + ) == expected + assert (arch.hidden_size, arch.num_attention_heads) == (48, 2) + + +@pytest.mark.parametrize( + "overrides,error", + [ + ({"model_dim_a": 50, "axes_dims_a": (4, 4, 4)}, "model_dim_a"), + ({"time_dim_a": 16}, "time_dim_a"), + ], +) +def test_incompatible_audio_dimensions_are_rejected(overrides, error): + with pytest.raises(ValueError, match=error): + Kandinsky6ArchConfig( + model_dim=48, time_dim=32, axes_dims=(8, 8, 8), **overrides + ) + + +@pytest.mark.parametrize( + "source,target", + [ + ( + "video_text_transformer_blocks.0.feed_forward.net.0.proj.weight", + "video_text_transformer_blocks.0.feed_forward.mlp.fc_in.weight", + ), + ( + "video_text_transformer_blocks.3.attn.to_q.weight", + "video_text_transformer_blocks.3.self_attention.to_q.weight", + ), + ( + "video_time_embeddings.timestep_embedder.linear_1.weight", + "video_time_embeddings.in_layer.weight", + ), + ( + "visual_transformer_blocks.0.video_dec_block.feed_forward.net.2.bias", + "visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_out.bias", + ), + ], +) +def test_checkpoint_name_mapping(source, target): + map_name = get_param_names_mapping(Kandinsky6ArchConfig().param_names_mapping) + assert map_name(source)[0] == target + + +@pytest.mark.parametrize("n_grid", [1, 10], ids=["sft", "distilled"]) +def test_checkpoint_head_width_does_not_change_latent_width(n_grid): + config = Kandinsky6VideoAudioConfig() + config.update_model_arch( + { + "in_visual_dim": 16, + "out_visual_dim": 16 * n_grid, + "in_audio_dim": 40, + "out_audio_dim": 40 * n_grid, + "model_dim_a": 2048, + "ff_dim_a": 7168, + "fix_modulation": True, + "scale_factor": [1.0, 2.0, 2.0], + } + ) + arch = config.arch_config + assert (arch.in_channels, arch.out_channels, arch.num_channels_latents) == ( + 16, + 16 * n_grid, + 16, + ) + assert (arch.in_audio_dim, arch.out_audio_dim) == (40, 40 * n_grid) + assert tuple(arch.scale_factor) == (1.0, 2.0, 2.0) + + +@pytest.mark.parametrize("scaling_factor", [0.5302, 0.417]) +def test_audio_vae_scaling_factor_comes_from_the_checkpoint(scaling_factor): + config = Kandinsky6AudioVAEConfig() + config.update_model_arch( + { + "mode": "44k", + "sample_rate": 44100, + "downsample_factor": 1024, + "scaling_factor": scaling_factor, + } + ) + assert config.arch_config.scaling_factor == scaling_factor diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py new file mode 100644 index 0000000000..55ebab7d46 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_audio_vae_weight_norm.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GAP 2 regression: ``Kandinsky6AudioVAE`` must not re-renormalize an +already-loaded (final) checkpoint's audio-VAE Conv1d weights. + +The diffusers reference calls ``MMAudioVAE.remove_weight_norm()`` exactly +once, inside ``MMAudioVAE.__init__`` -- BEFORE ``from_pretrained`` overwrites +those freshly-constructed (random) weights with the checkpoint's real, +already-final values, so a real checkpoint's own weights are never +renormalized. The pre-fix SGLang port instead triggered the same +RMS-renormalization lazily, on the first ``decode()`` call -- i.e. AFTER a +real checkpoint's weights were loaded -- corrupting them (measured by the +audit at ~9.7% decoded-waveform rel-L2 error against the diffusers +reference). +""" + +from __future__ import annotations + +import torch + +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAEConfig, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAE, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import ( + MMAudioVAE, +) + +# A small, self-contained BigVGANV2 config (not a real checkpoint's) just so +# Kandinsky6AudioVAE.__init__ can build the whole module tree on CPU quickly; +# none of these tests call vocode()/wrapped_decode(), only the mel-level +# decode() where the audited bug lives. +_TINY_VOCODER_CONFIG = { + "resblock": "1", + "num_mels": 80, # matches mode="16k"'s data_dim + "upsample_rates": [2], + "upsample_kernel_sizes": [4], + "upsample_initial_channel": 8, + "resblock_kernel_sizes": [3], + "resblock_dilation_sizes": [[1, 1, 1]], + "activation": "snake", + "snake_logscale": False, +} + + +def _tiny_audio_vae_config() -> Kandinsky6AudioVAEConfig: + config = Kandinsky6AudioVAEConfig() + config.arch_config.mode = "16k" + config.arch_config.vocoder_config = dict(_TINY_VOCODER_CONFIG) + return config + + +def test_mmaudio_vae_remove_weight_norm_is_idempotent(): + torch.manual_seed(0) + vae = MMAudioVAE(mode="16k", need_encoder=False) + vae.remove_weight_norm() + latents = torch.randn(1, vae.embed_dim, 4) + mel_once = vae.decode(latents.clone()) + + # A second, redundant call must be a true no-op: it must not re-derive + # (and thus change) already-normalized weights. + vae.remove_weight_norm() + mel_twice = vae.decode(latents.clone()) + + assert torch.equal(mel_once, mel_twice) + + +def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load(): + """Simulates VAELoader's real flow: construct, then load_state_dict with + a checkpoint's own (already-final) weights -- decode() must use those + weights as-is, matching the diffusers reference's "already normalized, + no-op at inference" contract, not silently re-derive a different set of + weights from them. + """ + torch.manual_seed(0) + config = _tiny_audio_vae_config() + vae = Kandinsky6AudioVAE(config) + assert vae.vae._weights_normalized is True + + # `load_state_dict(vae.state_dict())` mirrors VAELoader's real call + # (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's + # own current weights standing in for "a checkpoint's own weights" -- + # exactly what happens when a real checkpoint's stored tensors are + # loaded: no code path scales or otherwise disturbs them afterward. + vae.load_state_dict(vae.state_dict()) + + latents = torch.randn(1, vae.vae.embed_dim, 4) + mel_a = vae.decode(latents.clone()) + + # An explicit extra call (this wrapper's own public API) must also be a + # no-op now, not a second renormalization of the loaded weights. + vae.remove_weight_norm() + mel_b = vae.decode(latents.clone()) + + assert torch.equal(mel_a, mel_b) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py new file mode 100644 index 0000000000..a0b4f9c16c --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_bf16_dtype.py @@ -0,0 +1,502 @@ +# SPDX-License-Identifier: Apache-2.0 +"""BF16 forward, attention dispatch, CUDA graph and cache lifetime regressions.""" + +from __future__ import annotations + +import importlib.util +import os +from types import SimpleNamespace + +import pytest +import torch + +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( + CacheDitConfig, + disable_cache_on_transformer, + enable_cache_on_transformer, + refresh_context_on_transformer, +) +from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend_context_manager, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( + Kandinsky6Attention, + Kandinsky6Modulation, + Kandinsky6QKNorm, + Kandinsky6TimeEmbeddings, + Kandinsky6Transformer3DModel, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.denoising import ( + Kandinsky6DenoisingStage, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils import precision + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + # Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which + # need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's + # fixture of the same name. + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29513", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def _randomize(module: torch.nn.Module, seed: int) -> None: + generator = torch.Generator().manual_seed(seed) + with torch.no_grad(): + for param in module.parameters(): + param.copy_(torch.randn(param.shape, generator=generator) * 0.05) + + +def test_time_embeddings_forward_under_bf16_linear_weights(): + # Minimal, pinpoint reproduction of the review comment: a bf16-weight + # Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not + # raise, and must return bf16 (the Linear weights' dtype). + module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8) + _randomize(module, seed=0) + module = module.to(torch.bfloat16) + + out = module(torch.tensor([0.0, 500.0, 999.0])) + + assert out.dtype == torch.bfloat16 + assert torch.isfinite(out.float()).all() + + +def test_modulation_forward_under_bf16_linear_weights(): + # Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the + # identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time + # embedding with bf16 weights, it must not raise either. + module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9) + module = module.to(torch.bfloat16) + + out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to + + assert out.dtype == torch.bfloat16 + assert torch.isfinite(out.float()).all() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA attention") +@pytest.mark.parametrize("backend", ["FA", "TORCH_SDPA", "SAGE_ATTN_3"]) +@pytest.mark.parametrize("lengths", [(7, None), (129, None), (193, 65), (17, 129)]) +@torch.no_grad() +def test_attention_backend_dispatch_and_repeated_forward(monkeypatch, backend, lengths): + if backend == "FA" and torch.cuda.get_device_capability()[0] == 12: + pytest.skip("the platform currently resolves FA to SDPA on SM12.x") + if backend == "SAGE_ATTN_3": + if importlib.util.find_spec("sageattn3") is None: + pytest.skip("requires the optional SageAttention3 extension") + if torch.cuda.get_device_capability() not in {(12, 0), (12, 1)}: + pytest.skip("upstream SageAttention3 requires SM120 or SM121") + monkeypatch.setattr( + precision._mixed_precision_state, + "state", + precision.MixedPrecisionState(param_dtype=torch.bfloat16), + ) + query_len, key_len = lengths + selected = AttentionBackendEnum[backend] + with global_force_attn_backend_context_manager(selected): + layer = Kandinsky6Attention( + 256, + 128, + Kandinsky6Transformer3DModel._supported_attention_backends, + kv_dim=128 if key_len is not None else None, + is_cross_attention=key_len is not None, + ).eval() + assert layer.attention.backend == selected, "must not silently fall back" + _randomize(layer, seed=419) + layer.query_norm.weight.fill_(1) + layer.key_norm.weight.fill_(1) + layer.cuda().bfloat16() + torch.manual_seed(420) + query = torch.randn(1, query_len, 256, device="cuda", dtype=torch.bfloat16) + key = ( + torch.randn(1, key_len, 128, device="cuda", dtype=torch.bfloat16) + if key_len is not None + else None + ) + original_query = query.clone() + original_key = key.clone() if key is not None else None + implementation = layer.attention.attn_impl + forward = implementation.forward + calls = 0 + + def counted_forward(*args, **kwargs): + nonlocal calls + calls += 1 + return forward(*args, **kwargs) + + monkeypatch.setattr(implementation, "forward", counted_forward) + with set_forward_context(current_timestep=0, attn_metadata=None): + first = layer(query, key) + repeated = layer(query, key) + assert calls == 2, "must execute the requested backend, not a masked bypass" + assert first.shape == query.shape and first.dtype == query.dtype + assert torch.isfinite(first).all() + torch.testing.assert_close(repeated, first, rtol=0, atol=0) + torch.testing.assert_close(query, original_query, rtol=0, atol=0) + if key is not None: + torch.testing.assert_close(key, original_key, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) +@pytest.mark.parametrize("width", [16, 64, 128, 256]) +@pytest.mark.parametrize("eps", [None, 1e-5]) +@torch.no_grad() +def test_qk_norm_exact_output_and_parameter_storage(dtype, width, eps): + torch.manual_seed(943) + norm = Kandinsky6QKNorm(width, eps=eps, device="cuda", dtype=dtype) + norm.weight.normal_() + reference = torch.nn.RMSNorm(width, eps=eps, device="cuda", dtype=dtype) + reference.load_state_dict(norm.state_dict()) + parameter = norm.weight + pointer = parameter.data_ptr() + for scale in (0.0, 0.001, 1.0, 100.0): + x = (torch.randn(4096, width * 2, device="cuda", dtype=dtype) * scale)[:, ::2] + expected = reference(x.float()).to(dtype) + actual = norm(x.float()).to(dtype) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert norm.weight is parameter + assert norm.weight.data_ptr() == pointer + assert norm.weight.dtype == dtype + + norm.cpu().cuda() + torch.testing.assert_close(norm(x.float()).to(dtype), expected, rtol=0, atol=0) + torch.testing.assert_close(norm.weight, reference.weight, rtol=0, atol=0) + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_qk_norm_autograd_preserves_native_math(device): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("requires CUDA") + torch.manual_seed(944) + norm = Kandinsky6QKNorm(128, device=device, dtype=torch.bfloat16) + reference = torch.nn.RMSNorm(128, device=device, dtype=torch.bfloat16) + with torch.no_grad(): + norm.weight.normal_() + reference.load_state_dict(norm.state_dict()) + x = torch.randn(7, 128, device=device, requires_grad=True) + ref_x = x.detach().clone().requires_grad_() + expected = reference(ref_x) + actual = norm(x) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + actual.sum().backward() + expected.sum().backward() + torch.testing.assert_close(norm.weight.grad, reference.weight.grad, rtol=0, atol=0) + torch.testing.assert_close(x.grad, ref_x.grad, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs") +@torch.no_grad() +def test_qk_norm_graph_replay_observes_weight_updates(): + norm = Kandinsky6QKNorm(128, device="cuda", dtype=torch.bfloat16) + x = torch.randn(17, 128, device="cuda", dtype=torch.bfloat16).float() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(3): + norm(x) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = norm(x).to(torch.bfloat16) + for _ in range(2): + norm.weight.add_(0.125) + graph.replay() + expected = torch.nn.functional.rms_norm( + x, norm.normalized_shape, norm.weight, norm.eps + ).to(torch.bfloat16) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + +def _tiny_multimodal_arch_kwargs() -> dict: + """Small but structurally real T2VA config: is_multimodal=True (every released + Kandinsky6 checkpoint), audio tower dims intentionally different from the video + tower's to exercise both ``Kandinsky6TimeEmbeddings``/``Kandinsky6Modulation`` + instances the fused block owns.""" + return dict( + model_dim=32, + time_dim=16, + ff_dim=64, + axes_dims=(8, 4, 4), + in_visual_dim=4, + out_visual_dim=4, + in_text_dim=8, + in_text_dim2=8, + in_audio_dim=4, + num_text_blocks=1, + num_visual_blocks=1, + patch_size=(1, 1, 1), + visual_cond=False, + visual_token_type_num_embeddings=0, + is_multimodal=True, + model_dim_a=16, + time_dim_a=16, + ff_dim_a=32, + axes_dims_a=(4, 2, 2), + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs") +@pytest.mark.parametrize("use_cfg", [False, True]) +@torch.no_grad() +def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_cfg): + args = get_global_server_args() + args.attention_backend = "fa" + args.pipeline_config = Kandinsky6TI2VAPipelineConfig() + config = Kandinsky6VideoAudioConfig() + config.update_model_arch(_tiny_multimodal_arch_kwargs()) + with torch.device("meta"): + dit = Kandinsky6Transformer3DModel(config, {}).eval() + torch.manual_seed(7) + dit.load_state_dict( + { + name: (torch.randn(param.shape, device="cuda") * 0.05).to(torch.bfloat16) + for name, param in dit.named_parameters() + }, + assign=True, + ) + dit.post_load_weights() + stage = Kandinsky6DenoisingStage(dit, scheduler=None) + batch = SimpleNamespace( + is_warmup=False, + do_classifier_free_guidance=use_cfg, + guidance_scale=5.0, + cfg_normalization=0.0, + guidance_rescale=0.0, + ) + inputs = dict( + transformer=dit, + batch=batch, + server_args=args, + step_index=0, + video_input=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16), + audio_input=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16), + t_expand=torch.tensor([500.0], device="cuda"), + visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)], + scale_factor=(1.0, 1.0, 1.0), + sparse_params=None, + visual_token_type_ids=None, + ) + + def policy(text_len): + return CFGPolicy( + branches=[ + CFGBranch( + name, + conditional, + { + "encoder_hidden_states": torch.randn( + 1, text_len, 8, device="cuda", dtype=torch.bfloat16 + ), + "pooled_projections": torch.randn( + 1, 8, device="cuda", dtype=torch.bfloat16 + ), + "text_rope_pos": torch.arange(text_len, device="cuda"), + }, + ) + for name, conditional in ( + ("conditional", True), + ("unconditional", False), + ) + if conditional or use_cfg + ] + ) + + policies = [policy(3), policy(3), policy(5)] + references = [ + stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies + ] + args.enable_breakable_cuda_graph = True + batch.is_warmup = True + stage._predict_joint_velocity(cfg_policy=policies[0], **inputs) + batch.is_warmup = False + runner = stage._bcg_runners[id(dit)] + replay = runner.replay + replay_calls = 0 + + def record_replay(entry, kwargs): + nonlocal replay_calls + replay_calls += 1 + return replay(entry, kwargs) + + monkeypatch.setattr(runner, "replay", record_replay) + try: + assert len(runner.entries) == 1, "eager fallback is not a capture test" + assert all(entry.num_segments > 0 for entry in runner.entries.values()) + outputs = [ + stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies + ] + assert replay_calls == (4 if use_cfg else 2) + # validate after every replay: neither CFG nor subsequent requests may + # overwrite a previously returned video/audio tensor + for actual, expected in zip(outputs, references, strict=True): + for tensor, reference in zip(actual, expected, strict=True): + torch.testing.assert_close(tensor, reference, rtol=0, atol=0) + assert len(runner.entries) == 1, "serving must not capture unseen text lengths" + finally: + runner.reset() + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason=( + "the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real " + "CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 " + "numerics are only meaningful on real hardware anyway" + ), +) +def test_kandinsky6_dit_real_bf16_forward_does_not_crash(): + """Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose + weights are bf16 -- the exact configuration the SGLang PR review comment flagged + as broken (a fp32 sinusoidal embedding feeding a bf16 Linear).""" + device = torch.device("cuda") + config = Kandinsky6VideoAudioConfig() + arch_kwargs = _tiny_multimodal_arch_kwargs() + config.update_model_arch(dict(arch_kwargs)) + dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval() + _randomize(dit, seed=1) + dit = dit.to(device=device, dtype=torch.bfloat16) + + batch, duration, height, width = 1, 2, 2, 2 + text_len, audio_len = 3, 5 + hidden_states = torch.randn( + batch, duration, height, width, 4, dtype=torch.bfloat16, device=device + ) + hidden_states_audio = torch.randn( + batch, audio_len, 4, dtype=torch.bfloat16, device=device + ) + encoder_hidden_states = torch.randn( + batch, text_len, 8, dtype=torch.bfloat16, device=device + ) + pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device) + # Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the + # caller), same as every real sampling loop -- exactly the dtype that used to + # mismatch the bf16 Linear. + timestep = torch.tensor([500.0], device=device) + visual_rope_pos = ( + torch.arange(duration, device=device), + torch.arange(height, device=device), + torch.arange(width, device=device), + ) + text_rope_pos = torch.arange(text_len, device=device) + + video_out, audio_out = dit( + hidden_states=hidden_states, + hidden_states_audio=hidden_states_audio, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + pooled_projections=pooled_projections, + visual_rope_pos=visual_rope_pos, + text_rope_pos=text_rope_pos, + ) + + assert video_out.shape == (batch, duration, height, width, 4) + assert audio_out.shape == (batch, audio_len, 4) + assert video_out.dtype == torch.bfloat16 + assert audio_out.dtype == torch.bfloat16 + assert torch.isfinite(video_out.float()).all() + assert torch.isfinite(audio_out.float()).all() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("separate_cfg", [False, True]) +@torch.no_grad() +def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg): + config = Kandinsky6VideoAudioConfig() + arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4} + config.update_model_arch(arch) + dit = Kandinsky6Transformer3DModel(config, {}).eval() + _randomize(dit, seed=11) + dit = dit.to(device="cuda", dtype=torch.bfloat16) + torch.manual_seed(13) + inputs = dict( + hidden_states=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16), + hidden_states_audio=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16), + encoder_hidden_states=torch.randn(1, 3, 8, device="cuda", dtype=torch.bfloat16), + pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16), + visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)], + text_rope_pos=torch.arange(3, device="cuda"), + ) + + def run_steps(): + outputs = [] + for step in range(3): + for branch in range(2 if separate_cfg else 1): + outputs.append( + dit( + **( + inputs + | { + "pooled_projections": inputs["pooled_projections"] + + branch + } + ), + timestep=torch.tensor([500.0 - step * 100], device="cuda"), + ) + ) + return outputs + + def assert_exact(actual, expected): + for pair, reference in zip(actual, expected, strict=True): + for tensor, target in zip(pair, reference, strict=True): + torch.testing.assert_close(tensor, target, rtol=0, atol=0) + + reference = run_steps() + enable_cache_on_transformer( + dit, + CacheDitConfig( + enabled=True, + num_inference_steps=3, + Fn_compute_blocks=1, + Bn_compute_blocks=1, + max_warmup_steps=1, + steps_computation_mask=[1, 1, 1], + steps_computation_policy="static", + ), + has_separate_cfg=separate_cfg, + ) + try: + assert_exact(run_steps(), reference) + cached_requests = [] + for _ in range(2): + refresh_context_on_transformer( + dit, + 3, + steps_computation_mask=[1, 0, 1], + steps_computation_policy="static", + ) + outputs = run_steps() + for video, audio in outputs: + assert torch.isfinite(video).all() + assert torch.isfinite(audio).all() + assert dit._context_manager.get_cached_steps() == [1] + if separate_cfg: + assert dit._context_manager.get_cfg_cached_steps() == [1] + cached_requests.append(outputs) + assert_exact(*cached_requests) + finally: + disable_cache_on_transformer(dit) + assert_exact(run_steps(), reference) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_clip_loading.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_clip_loading.py new file mode 100644 index 0000000000..29713d0019 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_clip_loading.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +import pytest +import torch +from safetensors.torch import load_file, save_file +from transformers import CLIPTextConfig, CLIPTextModel + +from sglang.multimodal_gen.configs.models.encoders import ( + CLIPTextConfig as NativeCLIPTextConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.krea2 import Krea2PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.stablediffusion3 import ( + StableDiffusion3PipelineConfig, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + ComponentLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import ( + TextEncoderLoader, +) +from sglang.multimodal_gen.runtime.models.encoders.clip import ( + CLIPTextModel as NativeCLIPTextModel, +) +from sglang.multimodal_gen.runtime.models.encoders.clip import ( + CLIPTextModelWithProjection, +) +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import ( + Kandinsky6TI2VAPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + _text_encoder_max_length_is_fixed, +) +from sglang.srt.runtime_context import get_parallel + + +@pytest.mark.parametrize( + "config_cls,fixed", + [ + (FluxPipelineConfig, [True, False]), + (Kandinsky6TI2VAPipelineConfig, [False, True]), + (Krea2PipelineConfig, [False, False]), + (StableDiffusion3PipelineConfig, [False, False, False]), + ], +) +def test_request_max_length_preserves_fixed_encoder_context(config_cls, fixed): + config = config_cls() + assert [ + _text_encoder_max_length_is_fixed(config, index) for index in range(len(fixed)) + ] == fixed + + +def test_clip_uses_shared_native_loader(): + assert "text_encoder_2" not in Kandinsky6TI2VAPipeline.component_loaders + assert "text_encoder_2" in Kandinsky6TI2VAPipelineConfig().native_only_components + for architecture in ("CLIPTextModel", "Qwen2_5_VLForConditionalGeneration"): + loader = ComponentLoader.for_component_type( + "text_encoder", "transformers", architecture + ) + assert type(loader) is TextEncoderLoader + + +@pytest.mark.parametrize("eos_token_id", [2, 31]) +@pytest.mark.parametrize("masked", [False, True]) +def test_clip_bare_and_prefixed_checkpoints_match(tmp_path, eos_token_id, masked): + torch.manual_seed(42) + device = "cuda" if torch.cuda.is_available() else "cpu" + reference = ( + CLIPTextModel( + CLIPTextConfig( + vocab_size=32, + hidden_size=16, + intermediate_size=32, + num_hidden_layers=2, + num_attention_heads=2, + max_position_embeddings=8, + bos_token_id=1, + eos_token_id=eos_token_id, + ) + ) + .to(device) + .eval() + ) + config = NativeCLIPTextConfig() + config.update_model_arch(reference.config.to_dict()) + config.post_diffusers_config_update() + weights = {name: value.cpu() for name, value in reference.state_dict().items()} + save_file(weights, str(tmp_path / "prefixed.safetensors")) + save_file( + {name.removeprefix("text_model."): value for name, value in weights.items()}, + str(tmp_path / "bare.safetensors"), + ) + ids = torch.tensor([[1, 5, 31, 31, 0], [1, 6, 7, 31, 0]], device=device) + mask = ( + torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]], device=device) + if masked + else None + ) + with get_parallel().override( + tp_size=1, tp_rank=0, tp_group=None, attn_tp_size=1, attn_tp_rank=0 + ): + models = [] + for checkpoint in ("prefixed", "bare"): + model = NativeCLIPTextModel(config).to(device).eval() + loaded = model.load_weights( + load_file(str(tmp_path / f"{checkpoint}.safetensors")).items() + ) + assert loaded == set(dict(model.named_parameters())) + models.append(model) + for name, value in models[0].state_dict().items(): + torch.testing.assert_close( + models[1].state_dict()[name], value, rtol=0, atol=0 + ) + with torch.no_grad(): + expected = reference(ids, attention_mask=mask, output_hidden_states=True) + outputs = [ + model(ids, attention_mask=mask, output_hidden_states=True) + for model in models + ] + for prefixed, bare, reference_output in ( + ( + outputs[0].last_hidden_state, + outputs[1].last_hidden_state, + expected.last_hidden_state, + ), + (outputs[0].pooler_output, outputs[1].pooler_output, expected.pooler_output), + ( + tuple(outputs[0].hidden_states), + tuple(outputs[1].hidden_states), + expected.hidden_states, + ), + ): + torch.testing.assert_close(prefixed, bare, rtol=0, atol=0) + torch.testing.assert_close(prefixed, reference_output, rtol=2e-5, atol=2e-5) + + +def test_bare_text_checkpoint_preserves_projection_weight(): + config = NativeCLIPTextConfig() + config.update_model_arch( + { + "hidden_size": 16, + "intermediate_size": 32, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "vocab_size": 32, + "projection_dim": 8, + } + ) + with get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0): + model = CLIPTextModelWithProjection(config) + weight = torch.randn_like(model.text_projection.weight) + assert model.load_weights([("text_projection.weight", weight)]) == { + "text_projection.weight" + } + torch.testing.assert_close(model.text_projection.weight, weight, rtol=0, atol=0) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py new file mode 100644 index 0000000000..6fdc167d7d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_image_conditioning_resize.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor +must aspect-preserving-resize-then-centre-crop, matching the diffusers +reference's ``encode_i2va_first_frame`` +(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height, +src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to +``(height, width)``), not stretch-resize the whole source image to the +target box (which distorts its aspect ratio whenever the source doesn't +already match the target aspect ratio). +""" + +from __future__ import annotations + +import numpy as np +import PIL.Image +import pytest +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import ( + Kandinsky6ImageEncodingStage, +) + + +def _gradient_image(width: int, height: int) -> PIL.Image.Image: + """A deterministic, per-pixel-distinguishable RGB image (not a flat + color), so resize/crop geometry is actually verifiable pixel-for-pixel. + """ + xs = np.linspace(0, 255, width, dtype=np.uint8) + ys = np.linspace(0, 255, height, dtype=np.uint8) + grid = np.zeros((height, width, 3), dtype=np.uint8) + grid[..., 0] = xs[None, :] + grid[..., 1] = ys[:, None] + grid[..., 2] = 128 + return PIL.Image.fromarray(grid, mode="RGB") + + +@pytest.mark.parametrize( + "src_h,src_w,height,width", + [ + (1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case) + (1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case) + (1024, 1536, 512, 768), # already 3:2 -- matching aspect + ], +) +def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width): + # encode_i2va_first_frame: scale = min(src_h/height, src_w/width); + # resize to (int(src_h/scale), int(src_w/scale)), then centre-crop. + scale = min(src_h / height, src_w / width) + expected = (int(src_h / scale), int(src_w / scale)) + assert ( + Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width) + == expected + ) + # The resize always covers the target box on both axes (never smaller). + assert expected[0] >= height + assert expected[1] >= width + + +@pytest.mark.parametrize( + "src_w,src_h,min_mean_abs_diff", + [ + (1920, 1080, 0.015), # 16:9 landscape source (audit's own case) + (1080, 1920, 0.05), # 9:16 portrait source (audit's own case) + ], +) +def test_mismatched_aspect_pil_image_is_cropped_not_stretched( + src_w, src_h, min_mean_abs_diff +): + # The model's documented default target resolution (512x768, 2:3) + # against realistic phone/webcam source aspect ratios. + height, width = 512, 768 + image = _gradient_image(src_w, src_h) + + produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width) + assert produced.shape == (1, 3, height, width) + + # Independent oracle: the diffusers reference's own aspect-preserving + # resize + centre-crop, computed directly via PIL, without calling any + # of the code under test. + scale = min(src_h / height, src_w / width) + new_h, new_w = int(src_h / scale), int(src_w / scale) + resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR) + top, left = (new_h - height) // 2, (new_w - width) // 2 + cropped = resized.crop((left, top, left + width, top + height)) + expected_np = np.array(cropped).astype(np.float32) / 255.0 + expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0 + + assert torch.allclose(produced[0], expected, atol=2.0 / 255.0) + + # The deleted stretch-resize procedure (plain `.resize((width, height))`, + # no crop) produces a materially different image whenever the source + # aspect ratio doesn't already match the target -- proving this is a + # real behavior change, not a no-op refactor. + stretched = image.resize((width, height), resample=PIL.Image.BILINEAR) + stretched_np = np.array(stretched).astype(np.float32) / 255.0 + stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0 + mean_abs_diff = (produced[0] - stretched_t).abs().mean().item() + assert mean_abs_diff > min_mean_abs_diff + + +def test_matching_aspect_pil_image_resize_only_agrees_with_stretch(): + # A sanity check on the oracle itself: when the source already matches + # the target aspect ratio, cover-resize-then-crop and plain stretch + # resize agree closely (no crop actually removes anything) -- so the + # large the divergence asserted above is specifically an aspect-ratio + # effect, not a general property of the two code paths. + height, width = 64, 96 + src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3... + # (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2) + image = _gradient_image(src_w, src_h) + + produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width) + stretched = image.resize((width, height), resample=PIL.Image.BILINEAR) + stretched_np = np.array(stretched).astype(np.float32) / 255.0 + stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0 + + mean_abs_diff = (produced[0] - stretched_t).abs().mean().item() + assert mean_abs_diff < 0.02 + + +def test_tensor_input_uses_cover_crop_geometry_too(): + height, width = 64, 96 + src_h, src_w = 108, 192 + tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1] + + produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width) + assert produced.shape[-2:] == (height, width) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py new file mode 100644 index 0000000000..d97157e755 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_layerwise_offload.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the declared offload groups, not just component selection.""" + +import sys +from types import SimpleNamespace + +import pytest +import torch + +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAEConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend_context_manager, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) +from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( + Kandinsky6Transformer3DModel, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import ( + Kandinsky6AudioVAE, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import get_global_server_args +from sglang.multimodal_gen.runtime.utils import precision +from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import ( + TINY_KVAE, + TINY_LU_MODEL, +) +from sglang.srt.utils.network import get_free_port_below_ephemeral + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") + + +@pytest.fixture(autouse=True) +def single_gpu(monkeypatch, default_global_server_args): + if not model_parallel_is_initialized(): + port = get_free_port_below_ephemeral() + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + monkeypatch.setenv(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + monkeypatch.setattr( + precision._mixed_precision_state, + "state", + precision.MixedPrecisionState( + param_dtype=torch.bfloat16, reduce_dtype=torch.float32 + ), + ) + args = get_global_server_args() + args.attention_backend = "fa" + args.pipeline_config = Kandinsky6TI2VAPipelineConfig() + with global_force_attn_backend_context_manager(AttentionBackendEnum.FA): + yield + + +def _dit(multimodal): + config = Kandinsky6VideoAudioConfig() + config.update_model_arch( + dict( + model_dim=128, + time_dim=32, + ff_dim=256, + axes_dims=(32, 48, 48), + in_visual_dim=4, + out_visual_dim=4, + in_text_dim=16, + in_text_dim2=8, + in_audio_dim=4, + out_audio_dim=4, + num_text_blocks=2, + num_visual_blocks=2, + patch_size=(1, 1, 1), + visual_cond=False, + visual_token_type_num_embeddings=0, + is_multimodal=multimodal, + model_dim_a=128, + time_dim_a=32, + ff_dim_a=256, + axes_dims_a=(32, 48, 48), + ca_rope=True, + cross_gates=True, + fix_modulation=True, + ) + ) + model = Kandinsky6Transformer3DModel(config, {}) + # TP-aware linears can use uninitialized storage before checkpoint loading + with torch.no_grad(): + for parameter in model.parameters(): + parameter.normal_(std=0.02) + model.to(device="cuda", dtype=torch.bfloat16).eval() + inputs = dict( + hidden_states=torch.randn(1, 2, 3, 3, 4, device="cuda", dtype=torch.bfloat16), + encoder_hidden_states=torch.randn( + 1, 7, 16, device="cuda", dtype=torch.bfloat16 + ), + pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16), + timestep=torch.tensor([500.0], device="cuda"), + visual_rope_pos=[torch.arange(n, device="cuda") for n in (2, 3, 3)], + text_rope_pos=torch.arange(7, device="cuda"), + ) + if multimodal: + inputs["hidden_states_audio"] = torch.randn( + 1, 5, 4, device="cuda", dtype=torch.bfloat16 + ) + groups = ( + ["video_text_transformer_blocks", "audio_text_transformer_blocks"] + if multimodal + else ["text_transformer_blocks"] + ) + return model, lambda: model(**inputs), groups + ["visual_transformer_blocks"] + + +def _audio_vae(): + config = Kandinsky6AudioVAEConfig() + config.arch_config.mode = "16k" + config.arch_config.vocoder_config = dict( + resblock="1", + num_mels=80, + upsample_rates=[2], + upsample_kernel_sizes=[4], + upsample_initial_channel=8, + resblock_kernel_sizes=[3], + resblock_dilation_sizes=[[1, 1, 1]], + activation="snake", + snake_logscale=False, + ) + model = Kandinsky6AudioVAE(config).to("cuda").eval() + mel = torch.randn(1, 80, 8, device="cuda") + + def forward(): + latent = model.vae.encode(mel).mean + decoded = model.decode(latent) + return latent, decoded, model.vocode(decoded) + + groups = [f"vae.encoder.down.{i}.block" for i in range(3)] + groups += [f"vae.decoder.up.{i}.block" for i in range(3)] + return model, forward, groups + ["vocoder.ups.0", "vocoder.resblocks"] + + +def _sr_vae(): + config = Kandinsky6SRVAEConfig() + config.update_model_arch( + dict( + vae_type="video-kvae", + encoder_config=dict(TINY_KVAE, in_channels=3), + decoder_config=dict(TINY_KVAE, out_ch=3), + ) + ) + model = Kandinsky6SRVAE(config).to("cuda").eval() + pixels = torch.randn(1, 3, 33, 32, 32, device="cuda") + + def forward(): + latent, segments = model.encode(pixels) + assert segments == [17, 16] + return latent, model.decode(latent).sample + + groups = [f"encoder.down.{i}.block" for i in range(5)] + groups += [f"decoder.up.{i}.block" for i in range(5)] + return model, forward, groups + + +def _upscaler(): + model = ( + Kandinsky6SRLatentUpscalerBank( + [{"target_scale": scale, "model": dict(TINY_LU_MODEL)} for scale in (2, 4)] + ) + .to("cuda") + .eval() + ) + latent = torch.randn(1, 4, 3, 3, 3, device="cuda") + + def forward(): + # exercise both branches of every loaded entry, including width changes + return tuple( + entry(latent, scale) for entry in model._models for scale in (2, 4) + ) + + groups = [ + f"_models.{index}.{group}" + for index in range(2) + for group in ( + "pre_blocks", + "mid_blocks", + "post_blocks", + "x2_branch.adapter", + "x2_branch.mid_blocks", + "x2_branch.blocks", + ) + ] + return model, forward, groups + + +@pytest.mark.parametrize( + "component", ["video-dit", "joint-dit", "audio-vae", "sr-vae", "upscaler"] +) +@pytest.mark.parametrize("resident_layers", [0, 1]) +@torch.no_grad() +def test_layerwise_groups_preserve_repeated_forwards(component, resident_layers): + torch.manual_seed(13) + if component.endswith("dit"): + model, forward, groups = _dit(component == "joint-dit") + elif component == "audio-vae": + model, forward, groups = _audio_vae() + elif component == "sr-vae": + model, forward, groups = _sr_vae() + else: + model, forward, groups = _upscaler() + assert isinstance(model, LayerwiseOffloadableModuleMixin) + assert set(groups) <= set(model.layer_names) + assert model.layerwise_offload_dit_group_enabled == component.endswith("dit") + state = {name: value.cpu().clone() for name, value in model.state_dict().items()} + with ( + torch.inference_mode(), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + expected = forward() + model.configure_layerwise_offload( + SimpleNamespace( + performance_mode="speed", + pin_cpu_memory=True, + layerwise_tuning_for=lambda *args, **kwargs: ( + 1, + resident_layers, + "leading", + "forward", + ), + ) + ) + assert { + manager.layers_attr_str for manager in model.layerwise_offload_managers + } == set(groups) + for _ in range(2): + model.prepare_for_next_req() + torch.testing.assert_close(forward(), expected, rtol=0, atol=0) + for manager in model.layerwise_offload_managers: + assert manager._last_forwarded_layer == manager.num_layers - 1 + manager.release_all() + model.disable_offload() + torch.testing.assert_close(forward(), expected, rtol=0, atol=0) + for name, value in model.state_dict().items(): + torch.testing.assert_close(value.cpu(), state[name], rtol=0, atol=0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, *sys.argv[1:]])) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_lora.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_lora.py new file mode 100644 index 0000000000..8f649650cf --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_lora.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Load real PEFT safetensors through Kandinsky6's native adapter mapping.""" + +from collections import defaultdict +from types import SimpleNamespace + +import pytest +import torch +from safetensors.torch import save_file + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import ( + Kandinsky6TI2VAPipeline, +) + + +@pytest.mark.parametrize( + "prefix", ["", "transformer.", "base_model.model.transformer."] +) +@pytest.mark.parametrize( + "source,target", + [ + ( + "visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj", + "visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_in", + ), + ( + "visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2", + "visual_transformer_blocks.0.audioT.feed_forward.mlp.fc_out", + ), + ( + "video_text_transformer_blocks.1.feed_forward.net.0.proj", + "video_text_transformer_blocks.1.feed_forward.mlp.fc_in", + ), + ( + "audio_text_transformer_blocks.1.attn.to_query", + "audio_text_transformer_blocks.1.self_attention.to_query", + ), + ( + "video_time_embeddings.timestep_embedder.linear_1", + "video_time_embeddings.in_layer", + ), + ( + "audio_time_embeddings.timestep_embedder.linear_2", + "audio_time_embeddings.out_layer", + ), + ( + "visual_transformer_blocks.0.videoT.self_attention.to_key", + "visual_transformer_blocks.0.videoT.self_attention.to_key", + ), + ], +) +def test_peft_adapter_maps_to_native_layers(tmp_path, prefix, source, target): + pipeline = object.__new__(Kandinsky6TI2VAPipeline) + pipeline.server_args = SimpleNamespace( + pipeline_config=Kandinsky6TI2VAPipelineConfig(), lora_path=None + ) + pipeline.modules = { + "transformer": SimpleNamespace( + param_names_mapping={}, lora_param_names_mapping={} + ) + } + pipeline.device = torch.device("cpu") + pipeline.lora_adapters = defaultdict(dict) + pipeline.loaded_adapter_paths = {} + pipeline.loaded_adapter_alphas = {} + values = { + "lora_A": torch.arange(16, dtype=torch.float32).reshape(2, 8), + "lora_B": torch.arange(24, dtype=torch.float32).reshape(12, 2), + "alpha": torch.tensor(4.0), + } + path = tmp_path / "adapter.safetensors" + save_file( + { + f"{prefix}{source}.{key}.default.weight" + if key != "alpha" + else f"{prefix}{source}.alpha": value + for key, value in values.items() + }, + path, + ) + pipeline.load_lora_adapter(str(path), "probe", rank=0) + actual = pipeline.lora_adapters["probe"] + assert set(actual) == {f"{target}.{key}" for key in values} + for key, expected in values.items(): + torch.testing.assert_close(actual[f"{target}.{key}"], expected, rtol=0, atol=0) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py new file mode 100644 index 0000000000..3a25fe270f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py @@ -0,0 +1,346 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pipeline wiring, residency precision and joint video/audio denoising.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6 import ( + Kandinsky6TI2VASamplingParams, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy +from sglang.multimodal_gen.runtime.managers.memory_managers import ( + component_residency_strategies, +) +from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( + Kandinsky6RoPE1D, + Kandinsky6RoPE3D, +) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import ( + Kandinsky6TI2VAPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import ( + Kandinsky6AudioDecodingStage, + Kandinsky6DecodingStage, + Kandinsky6DenoisingStage, + Kandinsky6ImageEncodingStage, + Kandinsky6LatentPreparationStage, + denoising, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import ( + TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.server_args import get_global_server_args + +EXPECTED_REQUIRED_MODULES = [ + "scheduler", + "text_encoder", + "text_encoder_2", + "tokenizer", + "tokenizer_2", + "transformer", + "vae", + "audio_vae", +] + +EXPECTED_STAGE_ORDER = [ + ("input_validation_stage", InputValidationStage), + ("text_encoding_stage", TextEncodingStage), + ("timestep_preparation_stage", TimestepPreparationStage), + ("latent_preparation_stage", Kandinsky6LatentPreparationStage), + ("image_encoding_stage", Kandinsky6ImageEncodingStage), + ("denoising_stage", Kandinsky6DenoisingStage), + # Audio decodes before video, matching the diffusers reference -- see + # Kandinsky6TI2VAPipeline.create_pipeline_stages's comment. + ("audio_decoding_stage", Kandinsky6AudioDecodingStage), + ("decoding_stage", Kandinsky6DecodingStage), +] + + +def _make_pipeline() -> Kandinsky6TI2VAPipeline: + pipeline = object.__new__(Kandinsky6TI2VAPipeline) + pipeline.modules = { + name: MagicMock(name=name) + for name in Kandinsky6TI2VAPipeline._required_config_modules + } + pipeline._stages = [] + pipeline._stage_name_mapping = {} + pipeline._disagg_role = RoleType.MONOLITHIC + return pipeline + + +def test_pipeline_identity_and_required_modules(): + assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline" + assert Kandinsky6TI2VAPipeline.is_video_pipeline is True + assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES + # No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder + # and the BigVGAN vocoder in one checkpoint component. + assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules + + +def test_disagg_role_rejects_non_monolithic_deployment(): + pipeline = object.__new__(Kandinsky6TI2VAPipeline) + + with pytest.raises(ValueError, match="monolithic"): + pipeline.validate_disagg_role(RoleType.ENCODER) + with pytest.raises(ValueError, match="monolithic"): + pipeline.validate_disagg_role(RoleType.DENOISER) + + # MONOLITHIC is the only accepted role -- no exception raised. + pipeline.validate_disagg_role(RoleType.MONOLITHIC) + + +def test_create_pipeline_stages_produces_documented_order(): + pipeline = _make_pipeline() + server_args = MagicMock() + + Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args) + + actual_types = [type(stage) for stage in pipeline.stages] + expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER] + assert actual_types == expected_types + + actual_names = list(pipeline._stage_name_mapping.keys()) + expected_names = [name for name, _ in EXPECTED_STAGE_ORDER] + assert actual_names == expected_names + + # get_module wiring: each stage received the mocked module it asked for. + denoising_stage = pipeline._stage_name_mapping["denoising_stage"] + assert denoising_stage.transformer is pipeline.modules["transformer"] + + audio_decoding_stage = pipeline._stage_name_mapping["audio_decoding_stage"] + assert audio_decoding_stage.audio_vae is pipeline.modules["audio_vae"] + + decoding_stage = pipeline._stage_name_mapping["decoding_stage"] + assert decoding_stage.vae is pipeline.modules["vae"] + + image_encoding_stage = pipeline._stage_name_mapping["image_encoding_stage"] + assert image_encoding_stage.vae is pipeline.modules["vae"] + + +def _pro_distill_scheduler() -> PiflowScheduler: + """The scheduler of kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers.""" + scheduler = PiflowScheduler( + nfe=None, + n_grid=10, + shift=5.0, + eps=1e-6, + final_step_size_scale=0.5, + num_policy_substeps=128, + ) + scheduler.set_timesteps(10) + return scheduler + + +@pytest.mark.parametrize("stage_name", ["denoising_stage", "audio_decoding_stage"]) +@pytest.mark.parametrize("precision", ["bf16", "fp32"]) +def test_component_residency_preserves_loaded_precision(stage_name, precision): + pipeline = _make_pipeline() + args = MagicMock() + args.pipeline_config.dit_precision = precision + args.pipeline_config.audio_vae_precision = precision + args.component_precisions = {"transformer": precision, "audio_vae": precision} + pipeline.create_pipeline_stages(args) + use = pipeline._stage_name_mapping[stage_name].component_uses(args)[0] + + # loaders own precision; movement must not downcast RoPE or audio buffers + assert use.target_dtype is None + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers") +def test_denoising_offload_roundtrip_preserves_fp32_rope(monkeypatch): + device = torch.device("cuda", torch.cuda.current_device()) + monkeypatch.setattr( + component_residency_strategies, "get_local_torch_device", lambda: device + ) + transformer = torch.nn.ModuleDict( + { + "projection": torch.nn.Linear(8, 8, dtype=torch.bfloat16), + "rope_1d": Kandinsky6RoPE1D(8), + "rope_3d": Kandinsky6RoPE3D((4, 4, 4)), + } + ).to(device) + buffers = {name: value.clone() for name, value in transformer.named_buffers()} + stage = Kandinsky6DenoisingStage(transformer, scheduler=MagicMock()) + use = stage.component_uses(get_global_server_args())[0] + + for _ in range(2): + transformer.cpu() + component_residency_strategies._module_to_local_device( + transformer, dtype=use.target_dtype + ) + assert next(transformer.parameters()).dtype == torch.bfloat16 + for name, value in transformer.named_buffers(): + assert value.device == device + torch.testing.assert_close(value, buffers[name], rtol=0, atol=0) + + +def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch): + config = Kandinsky6TI2VAPipelineConfig() + batch = SimpleNamespace( + do_classifier_free_guidance=True, + guidance_scale=5.0, + cfg_normalization=0.0, + guidance_rescale=0.0, + ) + policy = config.cfg_policy.build(batch, {}, {}, {}) + positive = torch.linspace(-1, 1, 256, dtype=torch.bfloat16) + negative = torch.linspace(-0.8, 0.9, 256, dtype=torch.bfloat16) + predictions = [(positive, negative), (negative, positive)] + expected = policy.combine(predictions, batch, 5.0, config) + legacy = CFGPolicy().combine(predictions, batch, 5.0, config, cfg_parallel=True) + assert any(not torch.equal(a, b) for a, b in zip(expected, legacy, strict=True)) + monkeypatch.setattr(denoising, "get_classifier_free_guidance_world_size", lambda: 2) + monkeypatch.setattr(denoising, "run_cfg_parallel", lambda *_args: predictions) + monkeypatch.setattr( + denoising, + "run_two_branch_cfg_parallel", + lambda *_args: pytest.fail("legacy all-reduce changes BF16 rounding"), + ) + stage = Kandinsky6DenoisingStage(MagicMock(), scheduler=MagicMock()) + actual = stage._predict_joint_velocity( + transformer=stage.transformer, + cfg_policy=policy, + batch=batch, + server_args=SimpleNamespace(pipeline_config=config, enable_cfg_parallel=True), + step_index=0, + video_input=positive, + audio_input=negative, + t_expand=torch.zeros(1), + visual_rope_pos=[], + scale_factor=(1.0, 1.0, 1.0), + sparse_params=None, + visual_token_type_ids=None, + ) + for result, reference in zip(actual, expected, strict=True): + torch.testing.assert_close(result, reference, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "guidance_scale", [5.0, 0.5, 1.001, float("nan"), float("inf")] +) +def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler( + guidance_scale, +): + """The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance, + so the run must not silently ignore a guidance the caller asked for; the + Diffusers pipeline raises the same way ("PiflowScheduler requires + guidance_weight=1.0").""" + scheduler = _pro_distill_scheduler() + batch = SimpleNamespace( + timesteps=scheduler.timesteps, + latents=torch.zeros(1, 1, 1, 1, 1), + audio_latents=torch.zeros(1, 1, 1), + scheduler=scheduler, + guidance_scale=guidance_scale, + ) + stage = Kandinsky6DenoisingStage(transformer=MagicMock(), scheduler=scheduler) + + with pytest.raises(ValueError, match=r"requires guidance_scale=1\.0"): + stage.forward(batch, MagicMock()) + + +@pytest.mark.parametrize("height,width", [(32, 32), (512, 768), (768, 512)]) +def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance( + monkeypatch, height, width +): + """Pro-distill: 10 steps at guidance 1.0. The DiT head is ``n_grid`` (= 10) times as + wide as the latents, the pi-Flow step folds it back, and the returned latents keep + the latent width. The negative prompt is never fed to the DiT.""" + monkeypatch.setattr( + denoising, "get_local_torch_device", lambda: torch.device("cpu") + ) + monkeypatch.setattr( + get_global_server_args().pipeline_config, "dit_precision", "fp32" + ) + video_channels, audio_channels, n_grid, text_len = 4, 6, 10, 7 + text_lens = [] + + class _FakeDiT(torch.nn.Module): + """A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage`` + now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks + ``self.transformer.modules()`` to infer the attention backend -- a plain + callable has no ``.modules()``.""" + + def forward(self, **kwargs): + assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0) + text_lens.append(kwargs["encoder_hidden_states"].shape[1]) + video, audio = kwargs["hidden_states"], kwargs["hidden_states_audio"] + video_velocity = torch.randn(*video.shape[:-1], n_grid * video_channels) + audio_velocity = torch.randn(*audio.shape[:-1], n_grid * audio_channels) + assert kwargs["return_dict"] is False + return video_velocity, audio_velocity + + fake_dit = _FakeDiT() + + scheduler = _pro_distill_scheduler() + # visual_cond: the DiT input carries the latents, a condition copy and a mask + video = torch.randn(1, 2, 4, 4, 2 * video_channels + 1) + batch = SimpleNamespace( + timesteps=scheduler.timesteps, + latents=video, + audio_latents=torch.randn(1, 5, audio_channels), + scheduler=scheduler, + guidance_scale=1.0, + do_classifier_free_guidance=False, + extra={}, + prompt_embeds=[torch.randn(1, text_len, 8)], + negative_prompt_embeds=[torch.randn(1, text_len + 2, 8)], + pooled_embeds=[torch.randn(1, 3)], + prompt_attention_mask=[torch.ones(1, text_len)], + height=height, + width=width, + image_latent=None, + is_warmup=True, + sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"), + ) + arch = SimpleNamespace( + attention_engine="auto", + in_visual_dim=video_channels, + patch_size=(1, 2, 2), + scale_factor=(1.0, 2.0, 2.0), + ) + pipeline_config = SimpleNamespace( + dit_config=SimpleNamespace(arch_config=arch), + dit_precision="fp32", + vae_config=SimpleNamespace( + arch_config=SimpleNamespace(spatial_compression_ratio=8) + ), + get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0], + # CFG is disabled in this test (guidance_scale=1.0, + # do_classifier_free_guidance=False), so the policy never needs a + # negative branch, but Kandinsky6DenoisingStage still reads + # `cfg_policy` / `get_classifier_free_guidance_scale` off the config + # unconditionally to build the (single-branch) CFGPolicy. + cfg_policy=CFGPolicy(), + get_classifier_free_guidance_scale=lambda _batch, guidance_scale: ( + guidance_scale + ), + ) + server_args = SimpleNamespace( + pipeline_config=pipeline_config, enable_cfg_parallel=False + ) + stage = Kandinsky6DenoisingStage(transformer=fake_dit, scheduler=scheduler) + + result = stage.forward(batch, server_args) + + assert text_lens == [text_len] * 10 + assert tuple(result.latents.shape) == (1, 2, 4, 4, video_channels) + assert tuple(result.audio_latents.shape) == (1, 5, audio_channels) + assert torch.isfinite(result.latents).all() + assert torch.isfinite(result.audio_latents).all() diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_registry.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_registry.py new file mode 100644 index 0000000000..85cad36b39 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_registry.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 model routing and request defaults, without Hub access.""" + +import json +from types import SimpleNamespace + +import pytest + +from sglang.multimodal_gen import registry +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6 import ( + Kandinsky6TI2VADistilledSamplingParams, + Kandinsky6TI2VASamplingParams, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import ( + Kandinsky6TI2VAPipeline, +) +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import ( + Kandinsky6SRPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req + +SFT = "kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers" +DISTILL = "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers" +TI2VA = ( + Kandinsky6TI2VAPipeline, + Kandinsky6TI2VAPipelineConfig, + Kandinsky6TI2VASamplingParams, +) +DISTILLED = ( + Kandinsky6TI2VAPipeline, + Kandinsky6TI2VAPipelineConfig, + Kandinsky6TI2VADistilledSamplingParams, +) +SR = (Kandinsky6SRPipeline, Kandinsky6SRPipelineConfig, Kandinsky6SRSamplingParams) + +ROUTES = [ + ("kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA), + (SFT, TI2VA), + (DISTILL, DISTILLED), + ("kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", SR), + ("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR), + ("/models/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA), + ("/models/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA), + ("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers/", DISTILLED), + ("/models/Kandinsky-6.0-VSR-5s-Diffusers", SR), + ("/models/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR), + ("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA), + ( + "/hub/models--kandinskylab--Kandinsky-6.0-Pro-sft-5s-Diffusers/snapshots/4f2a", + TI2VA, + ), + ( + "/hub/models--kandinskylab--Kandinsky-6.0-Pro-distill-5s-Diffusers/snapshots/4f2a", + DISTILLED, + ), +] + + +@pytest.fixture(autouse=True) +def clear_registry_cache(): + registry.get_model_info.cache_clear() + registry._get_config_info.cache_clear() + yield + registry.get_model_info.cache_clear() + registry._get_config_info.cache_clear() + + +@pytest.mark.parametrize("model_path,expected", ROUTES) +def test_model_routing(monkeypatch, model_path, expected): + pipeline, _, _ = expected + monkeypatch.setattr( + registry, + "maybe_download_model_index", + lambda _: {"_class_name": pipeline.pipeline_name}, + ) + assert registry.is_registered_diffusion_model_path(model_path) + info = registry.get_model_info(model_path, backend="sglang") + assert ( + info.pipeline_cls, + info.pipeline_config_cls, + info.sampling_param_cls, + ) == expected + + +@pytest.mark.parametrize( + "name,expected", + [ + (SFT, TI2VA), + (DISTILL, DISTILLED), + ("/models/kandinsky6-sr-native", SR), + ("/models/Kandinsky_6.0_SR_x2", SR), + ("Kandinsky6TI2VAPipeline", TI2VA), + ("Kandinsky6T2VAPipeline", TI2VA), + ("Kandinsky6SRPipeline", SR), + ("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA), + ("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR), + ], +) +def test_detectors_do_not_overlap(name, expected): + claimed = [ + registry._CONFIG_REGISTRY[model_id] + for model_id, detector in registry._MODEL_NAME_DETECTORS + if detector(name.lower()) + ] + assert len(claimed) == 1 + assert (claimed[0].pipeline_config_cls, claimed[0].sampling_param_cls) == expected[ + 1: + ] + + +@pytest.mark.parametrize("expected", [TI2VA, SR]) +def test_local_bundle_routing(tmp_path, expected): + pipeline, _, _ = expected + for component in ("transformer", "vae"): + (tmp_path / component).mkdir() + (tmp_path / "model_index.json").write_text( + json.dumps( + { + "_class_name": pipeline.pipeline_name, + "_diffusers_version": "0.37.0", + "transformer": ["diffusers", "X"], + "vae": ["diffusers", "Y"], + } + ) + ) + info = registry.get_model_info(str(tmp_path), backend="sglang") + assert ( + info.pipeline_cls, + info.pipeline_config_cls, + info.sampling_param_cls, + ) == expected + if expected == TI2VA: + info = registry.get_model_info( + str(tmp_path), + backend="sglang", + model_id="Kandinsky-6.0-Pro-distill-5s-Diffusers", + ) + assert info.sampling_param_cls is Kandinsky6TI2VADistilledSamplingParams + + +@pytest.mark.parametrize( + "model_path,overrides,steps,guidance", + [ + (SFT, {}, 50, 5.0), + (DISTILL, {}, 10, 1.0), + ("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers", {}, 10, 1.0), + (DISTILL, {"num_inference_steps": 8}, 8, 1.0), + (DISTILL, {"num_inference_steps": 50}, 50, 1.0), + (SFT, {"num_inference_steps": 25, "guidance_scale": 3.0}, 25, 3.0), + ], +) +def test_request_defaults_and_overrides( + monkeypatch, model_path, overrides, steps, guidance +): + monkeypatch.setattr( + registry, + "maybe_download_model_index", + lambda _: {"_class_name": "Kandinsky6TI2VAPipeline"}, + ) + args = SimpleNamespace( + backend="sglang", + model_id=None, + pipeline_class_name=None, + pipeline_config=Kandinsky6TI2VAPipelineConfig(), + output_path=None, + comfyui_mode=True, + num_gpus=1, + ) + params = SamplingParams.from_user_sampling_params_args( + model_path, args, prompt="a cat", **overrides + ) + assert (params.num_inference_steps, params.guidance_scale) == (steps, guidance) + assert (params.height, params.width, params.num_frames, params.fps) == ( + 512, + 768, + 121, + 24, + ) + assert Req(sampling_params=params).do_classifier_free_guidance == (guidance > 1) + + +@pytest.mark.parametrize( + "params_cls", + [Kandinsky6TI2VASamplingParams, Kandinsky6TI2VADistilledSamplingParams], +) +@pytest.mark.parametrize("num_gpus", [1, 2, 3, 4, 8]) +def test_request_frame_count_is_independent_of_gpu_count(params_cls, num_gpus): + params = params_cls() + params._adjust( + SimpleNamespace( + pipeline_config=Kandinsky6TI2VAPipelineConfig(), + output_path=None, + comfyui_mode=True, + num_gpus=num_gpus, + ) + ) + assert params.num_frames == 121 diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py new file mode 100644 index 0000000000..11a24e28b9 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +import pytest +import torch + +from sglang.multimodal_gen.runtime.models.dits import kandinsky6 + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("shape", [(1, 1, 1, 1), (2, 3, 5, 7), (1, 31, 32, 48)]) +def test_3d_rope_broadcast_matches_materialized_axes(device, shape, monkeypatch): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("requires CUDA") + rope = kandinsky6.Kandinsky6RoPE3D((64, 32, 32)).to(device) + batch, frames, height, width = shape + pos = [torch.arange(n * 2, device=device)[::2] for n in shape[1:]] + args = torch.cat( + [ + rope.args_0[pos[0]] + .view(1, frames, 1, 1, -1) + .repeat(batch, 1, height, width, 1), + (rope.args_1[pos[1]] / 2.0) + .view(1, 1, height, 1, -1) + .repeat(batch, frames, 1, width, 1), + (rope.args_2[pos[2]] / 3.16) + .view(1, 1, 1, width, -1) + .repeat(batch, frames, height, 1, 1), + ], + dim=-1, + ) + cosine, sine = args.cos(), args.sin() + expected = torch.stack([cosine, -sine, sine, cosine], dim=-1) + expected = expected.view(*expected.shape[:-1], 2, 2).unsqueeze(-4) + + def no_repeat(*args, **kwargs): + pytest.fail("RoPE axes should broadcast without materializing repeats") + + monkeypatch.setattr(torch.Tensor, "repeat", no_repeat) + actual = rope(shape, pos, (1.0, 2.0, 3.16)) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert actual.is_contiguous() + + +@pytest.mark.parametrize("strided", [False, True]) +def test_rotary_cpu_and_autograd(strided): + x = torch.randn(2, 17, 3, 64, requires_grad=True) + if strided: + x = x[:, ::2] + rope = torch.randn(1, x.shape[1], 1, 32, 2, 2) + expected = (rope * x.reshape(2, x.shape[1], 3, 32, 1, 2)).sum(-1).reshape_as(x) + actual = kandinsky6._apply_rotary(x, rope) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close( + torch.autograd.grad(actual.sum(), x, retain_graph=True)[0], + torch.autograd.grad(expected.sum(), x)[0], + rtol=0, + atol=0, + ) + + +@torch.no_grad() +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.version.hip is not None, + reason="CUDA required", +) +def test_rotary_uses_fused_kernel(monkeypatch): + original = kandinsky6.apply_matrix_rope + calls = [] + + def record(x, rope, dtype=None): + calls.append(x.shape) + return original(x, rope, dtype) + + monkeypatch.setattr(kandinsky6, "apply_matrix_rope", record) + x = torch.randn(1, 257, 8, 128, dtype=torch.bfloat16, device="cuda") + rope = torch.randn(1, 257, 1, 64, 2, 2, device="cuda") + expected = ( + (rope * x.float().reshape(1, 257, 8, 64, 1, 2)) + .sum(-1) + .reshape_as(x) + .to(x.dtype) + ) + assert torch.equal(kandinsky6._apply_rotary(x, rope), expected) + assert len(calls) == 1 + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("strided", [False, True]) +def test_rotary_cast_preserves_rounding_and_autograd(device, dtype, strided): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("requires CUDA") + x = torch.randn(2, 17, 3, 64, device=device, requires_grad=True) + if strided: + x = x[:, ::2] + rope = torch.randn(1, x.shape[1], 1, 32, 2, 2, device=device) + rounded = x.to(dtype).float().reshape(2, x.shape[1], 3, 32, 1, 2) + expected = (rope * rounded).sum(-1).reshape_as(x).to(dtype) + actual = kandinsky6._apply_rotary(x, rope, dtype) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close( + torch.autograd.grad(actual.sum(), x, retain_graph=True)[0], + torch.autograd.grad(expected.sum(), x)[0], + rtol=0, + atol=0, + ) + with torch.no_grad(): + torch.testing.assert_close( + kandinsky6._apply_rotary(x, rope, dtype), expected, rtol=0, atol=0 + ) + + +@torch.no_grad() +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.version.hip is not None, + reason="CUDA required", +) +def test_gate_sum_preserves_fp32_intermediates(): + x = torch.randn(2, 257, 128, dtype=torch.bfloat16, device="cuda") + update = torch.randn_like(x) + gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384] + expected = (x.float() + gate.float() * update.float()).to(x.dtype) + assert not torch.equal(expected, x + gate * update) + assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py new file mode 100644 index 0000000000..eb7e93a6b5 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Compare native Ulysses/Ring joint video-audio forwards with a single GPU.""" + +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from sglang.multimodal_gen.configs.models.dits.kandinsky6 import ( + Kandinsky6VideoAudioConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import ( + Kandinsky6Transformer3DModel, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import set_global_server_args +from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy +from sglang.srt.utils.network import get_free_port_below_ephemeral + + +def _forward(rank, world_size, ring, tp_size, port, output_dir, model_dim): + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + LOCAL_RANK=str(rank), + WORLD_SIZE=str(world_size), + ) + torch.cuda.set_device(rank) + set_mixed_precision_policy(torch.bfloat16, torch.float32) + global_force_attn_backend(AttentionBackendEnum.FA) + set_global_server_args( + SimpleNamespace( + attention_backend="fa", + attention_backend_config=None, + kv_gather_degree=1, + sp_split_auto=False, + pipeline_config=Kandinsky6TI2VAPipelineConfig(), + ) + ) + maybe_init_distributed_environment_and_model_parallel( + tp_size=tp_size, + sp_size=world_size // tp_size, + ulysses_degree=1 if ring else world_size // tp_size, + ring_degree=world_size // tp_size if ring else 1, + ) + config = Kandinsky6VideoAudioConfig() + config.update_model_arch( + dict( + model_dim=model_dim, + time_dim=32, + ff_dim=512, + axes_dims=(32, 48, 48), + in_visual_dim=4, + out_visual_dim=4, + in_text_dim=16, + in_text_dim2=8, + in_audio_dim=4, + out_audio_dim=4, + num_text_blocks=1, + num_visual_blocks=2, + patch_size=(1, 1, 1), + visual_cond=False, + visual_token_type_num_embeddings=0, + is_multimodal=True, + model_dim_a=model_dim, + time_dim_a=32, + ff_dim_a=256, + axes_dims_a=(32, 48, 48), + ca_rope=True, + cross_gates=True, + fix_modulation=True, + ) + ) + model = Kandinsky6Transformer3DModel(config, {}).eval() + generator = torch.Generator().manual_seed(7) + weights_path = Path(output_dir) / "weights.pt" + with torch.no_grad(): + if world_size == 1: + for parameter in model.parameters(): + parameter.copy_( + torch.randn(parameter.shape, generator=generator) * 0.02 + ) + torch.save(model.state_dict(), weights_path) + else: + weights = torch.load(weights_path, weights_only=True) + for name, parameter in model.named_parameters(): + weight_loader = parameter.__dict__.get("weight_loader") + if weight_loader is None: + parameter.copy_(weights[name]) + else: + weight_loader(parameter, weights[name]) + model = model.to(device=rank, dtype=torch.bfloat16) + + for frames in (2, 3): + # odd visual sequence length exercises masked tail padding; audio stays odd + generator.manual_seed(frames) + + def tensor(*shape): + return torch.randn(*shape, generator=generator).to( + device=rank, dtype=torch.bfloat16 + ) + + with ( + torch.no_grad(), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + video, audio = model( + hidden_states=tensor(1, frames, 3, 3, 4), + hidden_states_audio=tensor(1, 5, 4), + encoder_hidden_states=tensor(1, 7, 16), + pooled_projections=tensor(1, 8), + timestep=torch.tensor([500.0], device=rank), + visual_rope_pos=[torch.arange(n, device=rank) for n in (frames, 3, 3)], + text_rope_pos=torch.arange(7, device=rank), + ) + assert video.shape == (1, frames, 3, 3, 4) + assert audio.shape == (1, 5, 4) + assert torch.isfinite(video).all() and torch.isfinite(audio).all() + path = Path(output_dir) / f"{frames}.pt" + if world_size == 1: + torch.save((video.cpu(), audio.cpu()), path) + else: + reference = torch.load(path, weights_only=True, map_location=f"cuda:{rank}") + for actual, expected in zip((video, audio), reference, strict=True): + torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002) + dist.destroy_process_group() + + +def _launch(world_size, ring, output_dir, tp_size=1, model_dim=256): + port = get_free_port_below_ephemeral() + mp.spawn( + _forward, + args=(world_size, ring, tp_size, port, output_dir, model_dim), + nprocs=world_size, + ) + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs") +def test_joint_video_audio_sequence_parallel(tmp_path): + _launch(1, False, str(tmp_path)) + _launch(2, False, str(tmp_path)) + _launch(2, True, str(tmp_path)) + _launch(2, False, str(tmp_path), tp_size=2) + + +@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs") +def test_joint_video_audio_tensor_sequence_parallel(tmp_path): + _launch(1, False, str(tmp_path), model_dim=512) + for tp_size in (1, 2, 4): + _launch(4, False, str(tmp_path), tp_size=tp_size, model_dim=512) + _launch(4, True, str(tmp_path), tp_size=2, model_dim=512) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py new file mode 100644 index 0000000000..fa898bb575 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_config.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Kandinsky 6 SR checkpoint configuration and CLI request validation.""" + +import argparse +import re + +import pytest + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRArchConfig, + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6 import ( + Kandinsky6TI2VASamplingParams, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping + +TINY_DIT = dict( + in_visual_dim=4, + in_text_dim=8, + in_text_dim2=8, + time_dim=16, + out_visual_dim=4, + patch_size=[1, 2, 2], + model_dim=32, + ff_dim=64, + num_text_blocks=0, + num_visual_blocks=2, + axes_dims=[8, 4, 4], + visual_cond=False, + instruct_type="noise", + attention_params={"512": {"type": "flash"}}, + use_text=False, +) +TINY_PIFLOW = dict( + nfe=2, + num_policy_substeps=8, + final_step_size_scale=0.5, + shift=5.0, + n_grid=3, + eps=1e-6, +) +SR_PARAMS = dict( + scale_factor={"512": [1.0, 1.0, 1.0]}, + visual_size=[512], + scheduler_scale=5.0, + lq_noise_scale=0.7, + lq_noise_type="ddpm", + lq_channel_noise_scale=0.0, + cap_noise_timestep=False, + fps=24, +) + + +def flat_config(**changes): + """A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied.""" + flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3) + flat.update( + piflow_nfe=2, + piflow_num_policy_substeps=8, + piflow_final_step_size_scale=0.5, + piflow_shift=5.0, + piflow_eps=1e-6, + attribute_overrides={"instruct_type": "noise"}, + sr_visual_size=[512], + sr_scale_factor={"512": [1.0, 1.0, 1.0]}, + sr_scheduler_scale=5.0, + sr_lq_noise_scale=0.7, + sr_lq_noise_type="ddpm", + sr_lq_channel_noise_scale=0.0, + sr_cap_noise_timestep=False, + sr_fps=24, + ) + flat.update(changes) + return flat + + +def official_config(**changes): + """An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head + width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys. + """ + official = { + key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",) + } + official.update( + out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"], + attribute_overrides=None, + sr_params=dict(SR_PARAMS), + ) + official.update(changes) + return official + + +def parse(flat): + config = Kandinsky6SRDitConfig() + config.update_model_arch(flat) + return config.arch_config + + +# --------------------------------------------------------------------------- # +# DiT arch config +# --------------------------------------------------------------------------- # +def test_arch_config_parses_an_official_config_and_maps_checkpoint_names(): + arch = parse(official_config()) + assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler + assert arch.out_visual_dim == 12 and arch.attribute_overrides == {} + assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4) + assert (arch.hidden_size, arch.num_attention_heads) == (32, 2) + assert arch.sr_scale_factor == {"512": [1.0, 1.0, 1.0]} # JSON keys stay strings + assert (arch.sr_visual_size, arch.sr_scheduler_scale, arch.sr_fps) == ( + [512], + 5.0, + 24, + ) + + map_name = get_param_names_mapping(arch.param_names_mapping) + assert map_name("pooled_bias")[0] == "pooled_bias" + assert ( + map_name("visual_transformer_blocks.3.feed_forward.net.0.proj.weight")[0] + == "visual_transformer_blocks.3.feed_forward.mlp.fc_in.weight" + ) + assert ( + map_name("visual_transformer_blocks.0.feed_forward.net.2.weight")[0] + == "visual_transformer_blocks.0.feed_forward.mlp.fc_out.weight" + ) + assert ( + map_name("time_embeddings.timestep_embedder.linear_1.weight")[0] + == "time_embeddings.in_layer.weight" + ) + # the attention output projection is not a feed-forward layer and keeps its name + assert ( + map_name("visual_transformer_blocks.0.self_attention.out_layer.weight")[0] + == "visual_transformer_blocks.0.self_attention.out_layer.weight" + ) + + +def test_unknown_sr_params_keys_are_rejected(): + with pytest.raises(ValueError, match="sr_params has unsupported keys"): + parse(official_config(sr_params={"fps": 24, "new_knob": 1})) + + +@pytest.mark.parametrize( + "changes,error,flag", + [ + ({"use_text": True}, NotImplementedError, "use_text"), + ({"use_adapter": True}, NotImplementedError, "use_adapter"), + ({"use_lq_modulation": True}, NotImplementedError, "use_lq_modulation"), + ({"surprise_key": 1}, ValueError, "surprise_key"), + ({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"), + ({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"), + ({"piflow_eps": None}, ValueError, "piflow_eps"), + ({"patch_size": [2, 2, 2]}, ValueError, "patch_size"), + ({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"), + ], +) +def test_unsupported_or_unknown_config_fails_loudly_naming_the_key( + changes, error, flag +): + """A config the port cannot honor must never build a silently different model: + unsupported flags raise NotImplementedError naming the flag, unknown / inconsistent + keys raise ValueError (``update_model_arch`` alone would park unknown keys in + ``extra_attrs``).""" + with pytest.raises(error, match=flag): + parse(flat_config(**changes)) + + +def test_sparse_attention_request_is_detected_from_attention_params(): + assert parse(flat_config()).requested_sparse_attention() is None + nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}}) + assert parse(nabla).requested_sparse_attention() == "nabla" + # an override can switch it off again + overridden = flat_config( + attention_params={"512": {"type": "nabla"}}, + attribute_overrides={"attention_params": {"512": {"type": "flash"}}}, + ) + assert parse(overridden).requested_sparse_attention() is None + + +def test_default_arch_config_is_valid_and_text_free(): + """Registry / ``PipelineConfig.from_kwargs`` instantiate the default config.""" + arch = Kandinsky6SRArchConfig() + assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1 + + +# --------------------------------------------------------------------------- # +# VAE config +# --------------------------------------------------------------------------- # +def test_vae_config_parses_bundle_config_and_rejects_bad_ones(): + config = Kandinsky6SRVAEConfig() + config.update_model_arch( + dict( + vae_type="video-kvae", + encoder_config={"ch": 8}, + decoder_config={"ch": 8}, + scaling_factor=0.53, + spatial_factor=16, + temporal_factor=4, + ) + ) + arch = config.arch_config + assert arch.scaling_factor == 0.53 + assert (arch.spatial_compression_ratio, arch.temporal_compression_ratio) == (16, 4) + assert config.get_vae_scale_factor() == 16 + + with pytest.raises(ValueError, match="video-kvae"): + Kandinsky6SRVAEConfig().update_model_arch({"vae_type": "wan"}) + with pytest.raises(ValueError, match="surprise"): + Kandinsky6SRVAEConfig().update_model_arch({"surprise": 1}) + + +# --------------------------------------------------------------------------- # +# Pipeline config and sampling params +# --------------------------------------------------------------------------- # +def test_pipeline_config_is_text_free_monolithic_and_strict(): + config = Kandinsky6SRPipelineConfig() + config.check_pipeline_config() # the four per-encoder tuples must stay consistent + assert config.text_encoder_configs == () + assert config.supports_disaggregation() is False + # These components must load strictly and re-raise (no diffusers AutoModel fallback). + assert {"transformer", "vae", "latent_upscaler"} <= set( + config.native_only_components + ) + + +def test_sampling_params_validate_the_sr_knobs(): + params = Kandinsky6SRSamplingParams(video_path="clip.mp4") + assert (params.sr_resolution_scale, params.num_inference_steps) == (2.25, 4) + assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (1, 0.20) + assert params.seed == 42 and params.guidance_scale == 1.0 + + for bad in ( + {"sr_resolution_scale": 3}, + {"num_inference_steps": 0}, + {"sr_tiles_batch_size": 0}, + {"sr_tile_min_overlap": 1.0}, + {"sr_target_resize_mode": "crop"}, + {"sr_target_resolution": "big"}, + ): + with pytest.raises(ValueError): + Kandinsky6SRSamplingParams(video_path="clip.mp4", **bad) + for good in ("hd", "fullhd", "2k", "1280x720", "none"): + Kandinsky6SRSamplingParams(video_path="clip.mp4", sr_target_resolution=good) + + +def test_sr_flags_reach_the_sampling_params_through_the_generate_cli(): + """New SamplingParams fields are only CLI flags if the base ``add_cli_args`` declares + them; ``get_cli_args`` then filters by the resolved subclass' dataclass fields.""" + parser = argparse.ArgumentParser() + SamplingParams.add_cli_args(parser) + args = parser.parse_args( + [ + "--video-path", + "clip.mp4", + "--sr-resolution-scale", + "4", + "--num-inference-steps", + "3", + "--sr-tiles-batch-size", + "2", + "--sr-tile-min-overlap", + "0.3", + "--sr-target-resolution", + "fullhd", + "--sr-target-resize-mode", + "exact", + ] + ) + kwargs = Kandinsky6SRSamplingParams.get_cli_args(args) + params = Kandinsky6SRSamplingParams(**kwargs) + assert ( + params.video_path == ["clip.mp4"] and params.source_video_path() == "clip.mp4" + ) + assert (params.sr_resolution_scale, params.num_inference_steps) == (4.0, 3) + assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (2, 0.3) + assert (params.sr_target_resolution, params.sr_target_resize_mode) == ( + "fullhd", + "exact", + ) + # the sr_* knobs are ignored by other models' params instead of choking on them; the + # shared num_inference_steps flag, unlike those, is a core field every model gets. + assert "sr_tiles_batch_size" not in Kandinsky6TI2VASamplingParams.get_cli_args(args) + assert "num_inference_steps" in Kandinsky6TI2VASamplingParams.get_cli_args(args) + + +def test_request_needs_exactly_one_video_and_names_its_output_after_it(): + with pytest.raises(ValueError, match="--video-path"): + Kandinsky6SRSamplingParams()._validate_with_pipeline_config( + Kandinsky6SRPipelineConfig() + ) + with pytest.raises(ValueError, match="exactly one video"): + Kandinsky6SRSamplingParams(video_path=["a.mp4", "b.mp4"]).source_video_path() + + params = Kandinsky6SRSamplingParams(video_path="/data/my clip.mp4", prompt=" ") + params._set_output_file_name() + assert re.fullmatch(r"my_clip_sr2\.25x_\d{8}-\d{6}\.mp4", params.output_file_name) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py new file mode 100644 index 0000000000..8f413952ba --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_decode_parallel.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SR decode parallelism must preserve causal state and request isolation.""" + +import os +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_decode_parallel_group_coordinator, + get_dp_rank, + maybe_init_distributed_environment_and_model_parallel, +) +from sglang.multimodal_gen.runtime.layers.parallel_conv import ( + SpatialParallelConv3d, + disable_spatial_parallel_decode, + gather_and_trim_height, + split_height_for_parallel_decode, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import ( + Kandinsky6SRVAE, + _ChunkedConv3d, + _SpatialChunkedConv3d, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( + Kandinsky6SRDecodeStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_DENOISED_KEY, + SR_TILES_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + decode_chunks, +) +from sglang.multimodal_gen.runtime.server_args import set_global_server_args +from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_KVAE +from sglang.srt.utils.network import get_free_port_below_ephemeral + + +def _decode(rank, world_size, tp_size, dp_size, port, spatial): + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + LOCAL_RANK=str(rank), + WORLD_SIZE=str(world_size), + ) + torch.cuda.set_device(rank) + sp_size = world_size // tp_size // dp_size + maybe_init_distributed_environment_and_model_parallel( + tp_size=tp_size, sp_size=sp_size, ulysses_degree=sp_size, dp_size=dp_size + ) + group = get_decode_parallel_group_coordinator() + assert group.world_size == world_size // dp_size + spatial_active = spatial and group.world_size > 1 + if spatial_active: + torch.manual_seed(3 + get_dp_rank()) + conv = _ChunkedConv3d(2, 3, 3, padding=(0, 1, 1)).to(rank) + parallel_conv = _SpatialChunkedConv3d.from_conv(conv) + assert parallel_conv.weight is conv.weight + x = torch.randn(1, 2, 9, 9, 5, device=rank) + with torch.inference_mode(): + expected_conv = conv(x) + shard, height = split_height_for_parallel_decode( + x, x.shape[-2], group.world_size, group.rank_in_group + ) + # force different temporal chunk counts on uneven height shards + with patch( + "sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae._MAX_CONV_NUMEL", + 200, + ): + actual_conv = gather_and_trim_height(parallel_conv(shard), height) + torch.testing.assert_close(actual_conv, expected_conv) + with disable_spatial_parallel_decode(): + torch.testing.assert_close(parallel_conv(x), expected_conv) + config = Kandinsky6SRVAEConfig() + config.update_model_arch( + dict( + encoder_config=dict(TINY_KVAE, in_channels=3), + decoder_config=dict(TINY_KVAE, out_ch=3), + scaling_factor=0.5, + spatial_factor=2 ** (len(TINY_KVAE["ch_mult"]) - 1), + ) + ) + torch.manual_seed(17) + reference_vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16) + config = replace( + config, use_parallel_decode=spatial, parallel_decode_mode="spatial_shard" + ) + vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16) + vae.load_state_dict(reference_vae.state_dict()) + assert ( + any(isinstance(m, SpatialParallelConv3d) for m in vae.decoder.modules()) + == spatial_active + ) + pipeline_config = Kandinsky6SRPipelineConfig(vae_config=config) + args = SimpleNamespace(pipeline_config=pipeline_config, component_precisions={}) + set_global_server_args(args) + stage = Kandinsky6SRDecodeStage(vae) + stage.server_args = args + + with torch.inference_mode(): + for offload in (False, True): + if offload: + vae.configure_layerwise_offload( + SimpleNamespace( + performance_mode="speed", + pin_cpu_memory=True, + layerwise_tuning_for=lambda *args, **kwargs: ( + 1, + 0, + "leading", + "forward", + ), + ) + ) + first = None + # Uneven batches, idle ranks, two temporal segments, then a repeat. + for sizes, frames, height in ( + ((2, 2, 1), 9, 5), + ((1,), 1, 1), + ((2, 1), 5, 4), + ((2, 2, 1), 9, 5), + ): + generator = torch.Generator().manual_seed(7 + get_dp_rank()) + chunks = [ + torch.randn(size, frames, height, 3, 4, generator=generator) + for size in sizes + ] + reference_vae.prepare_for_next_req() + expected = decode_chunks( + chunks, + reference_vae, + scaling_factor=reference_vae.scaling_factor, + device=torch.device("cuda", rank), + ) + for parallel in (False, True): + config.use_parallel_tiling = parallel + vae.prepare_for_next_req() + request = Req(sampling_params=Kandinsky6SRSamplingParams()) + request.extra[SR_DENOISED_KEY] = chunks + with ( + patch.object(vae, "decode", wraps=vae.decode) as decode, + patch.object( + vae.decoder, "forward", wraps=vae.decoder.forward + ) as decoder, + ): + actual = stage.forward(request, args).extra[SR_TILES_KEY] + count = len( + range(group.rank_in_group, sum(sizes), group.world_size) + ) + assert decode.call_count == ( + count + if parallel and not spatial_active and sum(sizes) > 1 + else sum(sizes) + ) + local_height = height + if spatial_active and height >= group.world_size: + local_height = height // group.world_size + ( + group.rank_in_group < height % group.world_size + ) + assert all( + call.args[0].shape[-2] == local_height + for call in decoder.call_args_list + ) + assert len(actual) == len(expected) + for tile, reference in zip(actual, expected): + assert tile.device.type == "cpu" and tile.dtype == torch.uint8 + torch.testing.assert_close( + tile, reference, rtol=0, atol=2 if spatial_active else 0 + ) + if frames == 9: + if first is None: + first = actual + else: + for tile, reference in zip(actual, first): + torch.testing.assert_close(tile, reference, rtol=0, atol=0) + if offload: + vae.disable_offload() + dist.destroy_process_group() + + +@pytest.mark.parametrize( + "world_size,tp_size,dp_size", + [(1, 1, 1), (2, 2, 1), (2, 1, 1), (4, 2, 1), (4, 1, 2)], +) +@pytest.mark.parametrize("spatial", [False, True], ids=["tiles", "spatial"]) +def test_sr_parallel_decode(world_size, tp_size, dp_size, spatial): + if torch.cuda.device_count() < world_size: + pytest.skip(f"requires {world_size} CUDA GPUs") + mp.spawn( + _decode, + args=(world_size, tp_size, dp_size, get_free_port_below_ephemeral(), spatial), + nprocs=world_size, + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py new file mode 100644 index 0000000000..258ae0ee8e --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_http_request.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Check SR prompt admission and request fields without running a GPU scheduler. + +JSON uses TestClient; multipart tests call the parsed endpoint directly. The +full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads. +""" + +import asyncio +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.entrypoints.openai import video_api +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + resolve_sampling_params_cls, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import ( + _build_video_sampling_params, + _multipart_video_extras, +) + +_SR_EXTRA_FIELDS = frozenset( + { + "sr_resolution_scale", + "sr_tiles_batch_size", + "sr_tile_min_overlap", + "sr_target_resolution", + "sr_target_resize_mode", + } +) + + +def _fake_server_args(**overrides) -> SimpleNamespace: + base = dict( + pipeline_config=Kandinsky6SRPipelineConfig(), + pipeline_class_name="Kandinsky6SRPipeline", + backend="auto", + model_id=None, + model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", + served_model_name="kandinsky6-sr-test", + input_save_path=os.path.expanduser( + "~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads" + ), + output_path=os.path.expanduser( + "~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output" + ), + attention_backend_config={}, + enable_trace=False, + comfyui_mode=False, + num_gpus=1, + ) + base.update(overrides) + return SimpleNamespace(**base) + + +def _patched_global_server_args(server_args): + return patch( + "sglang.multimodal_gen.runtime.server_args.server_args._global_server_args", + server_args, + ) + + +# --------------------------------------------------------------------------- # +# The two declared pieces (SamplingParams-level) +# --------------------------------------------------------------------------- # +def test_prompt_optional_defaults_false_and_sr_overrides_it(): + """A generic pipeline keeps requiring ``prompt``; only the SR params opt out.""" + assert SamplingParams.prompt_optional is False + assert Kandinsky6SRSamplingParams.prompt_optional is True + + +@pytest.mark.parametrize( + "sampling_params_cls, status", + [(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)], +) +def test_json_prompt_requirement_matches_pipeline( + tmp_path, sampling_params_cls, status +): + app = FastAPI() + app.include_router(video_api.router) + server_args = _fake_server_args( + input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs") + ) + dispatch = AsyncMock() + with ( + _patched_global_server_args(server_args), + patch.object( + video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls + ), + patch.object(video_api, "_dispatch_job_async", dispatch), + TestClient(app) as client, + ): + response = client.post( + "/v1/videos", + json={"video_path": "unused.mp4", "sr_resolution_scale": 4}, + ) + assert response.status_code == status, response.text + if status == 200: + assert response.json()["status"] == "queued" + dispatch.assert_awaited_once() + _, batch = dispatch.call_args.args + assert batch.prompt == "" + assert batch.sampling_params.sr_resolution_scale == 4 + else: + dispatch.assert_not_called() + + +def test_sr_declares_its_cli_flags_as_multipart_extra_fields(): + assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS + # The base default (what every other model still gets) stays empty. + assert SamplingParams.video_request_extra_fields() == frozenset() + + +# --------------------------------------------------------------------------- # +# The real multipart-field allowlist (video_api._multipart_video_extras) +# --------------------------------------------------------------------------- # +def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class(): + raw_form = { + "video_path": "unused.mp4", + "sr_resolution_scale": "4", + "sr_tiles_batch_size": "3", + "not_a_declared_field": "dropped", + } + sr_extras = _multipart_video_extras( + raw_form, + extra_body=None, + extra_params=None, + sampling_params_cls=Kandinsky6SRSamplingParams, + ) + assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4" + assert sr_extras["sr_tiles_batch_size"] == 3 + assert "not_a_declared_field" not in sr_extras + + # A model without the override (the pre-fix behaviour) drops the sr_* fields too. + base_extras = _multipart_video_extras( + raw_form, + extra_body=None, + extra_params=None, + sampling_params_cls=SamplingParams, + ) + assert "sr_resolution_scale" not in base_extras + assert "sr_tiles_batch_size" not in base_extras + + +# --------------------------------------------------------------------------- # +# The registry-backed class resolution the admission gate reads +# --------------------------------------------------------------------------- # +def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline(): + server_args = _fake_server_args() + assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams + + +# --------------------------------------------------------------------------- # +# End-to-end request parsing: a prompt-less request field set -> sampling params +# --------------------------------------------------------------------------- # +def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields(): + """The real kwargs-building code (``_build_video_sampling_params``), given a + request built the way the multipart branch builds one for a prompt-optional + pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a + real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error.""" + from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + VideoGenerationsRequest, + ) + + server_args = _fake_server_args() + raw_form = { + "video_path": "unused.mp4", + "sr_resolution_scale": "4", + "sr_tiles_batch_size": "3", + } + extras = _multipart_video_extras( + raw_form, + extra_body=None, + extra_params=None, + sampling_params_cls=Kandinsky6SRSamplingParams, + ) + req = VideoGenerationsRequest( + prompt="", # what video_api.py substitutes for a missing prompt + video_path=extras.pop("video_path"), + **extras, + ) + + with _patched_global_server_args(server_args): + sampling_params = _build_video_sampling_params("gap1-request-id", req) + + assert isinstance(sampling_params, Kandinsky6SRSamplingParams) + assert sampling_params.prompt == "" + assert sampling_params.sr_resolution_scale == 4 + assert sampling_params.sr_tiles_batch_size == 3 + + +# --------------------------------------------------------------------------- # +# The full endpoint, called directly (see module docstring for why not TestClient) +# --------------------------------------------------------------------------- # +class _FakeMultipartRequest: + """Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly.""" + + def __init__(self, form: dict): + self.headers = {"content-type": "multipart/form-data; boundary=x"} + self._form = form + + async def form(self): + return self._form + + +def test_multipart_sr_request_without_prompt_is_accepted(): + """A multipart SR request with no ``prompt`` field must be admitted (no 400), the + ``sr_*`` fields must reach the queued request, and it must fail on the pre-fix + code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would + reject this before any of the rest of ``create_video`` ran.""" + server_args = _fake_server_args() + request = _FakeMultipartRequest( + {"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"} + ) + dispatched: list[tuple] = [] + + async def _fake_dispatch(job_id, batch, **kwargs): + dispatched.append((job_id, batch, kwargs)) + + async def run(): + with ( + _patched_global_server_args(server_args), + patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch), + ): + response = await video_api.create_video( + request, + prompt=None, # <-- the field under test: no prompt supplied + enhance_prompt=None, + task_type=None, + x264_preset=None, + perf_dump_path=None, + input_reference=None, + reference_url=None, + video_reference=None, + video_url=None, + video_path="unused.mp4", + model=None, + n=1, + num_outputs_per_prompt=None, + seconds=None, + size=None, + fps=None, + num_frames=None, + seed=None, + generator_device="cuda", + negative_prompt=None, + guidance_scale=None, + guidance_scale_2=None, + true_cfg_scale=None, + num_inference_steps=None, + max_sequence_length=None, + flow_shift=None, + enable_teacache=None, + enable_frame_interpolation=None, + frame_interpolation_exp=None, + frame_interpolation_scale=None, + frame_interpolation_model_path=None, + enable_upscaling=None, + upscaling_model_path=None, + upscaling_scale=None, + output_quality=None, + output_compression=None, + output_path=None, + extra_params=None, + extra_body=None, + ) + await asyncio.sleep(0) + return response + + response = asyncio.run(run()) + + assert isinstance(response, VideoResponse) + assert response.status == "queued" + assert len(dispatched) == 1 + _, batch, _ = dispatched[0] + assert batch.prompt == "" + assert batch.sampling_params.sr_resolution_scale == 4 + assert batch.sampling_params.sr_tiles_batch_size == 3 diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py new file mode 100644 index 0000000000..572ef58871 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_loading.py @@ -0,0 +1,467 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Strict loading, dtype and config contracts for tiny Kandinsky 6 SR components.""" + +import copy +import json +import os + +import pytest +import torch +from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL +from safetensors.torch import load_file, save_file + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.loader.component_loaders import ( + component_loader as component_loader_module, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.latent_upscaler_loader import ( + LatentUpscalerLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.vae_loader import VAELoader +from sglang.multimodal_gen.runtime.loader.fsdp_load import ( + load_model_from_full_model_state_dict, +) +from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping +from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( + Kandinsky6SRTransformer3DModel, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE + +TINY_DIT = dict( + in_visual_dim=4, + in_text_dim=8, + in_text_dim2=8, + time_dim=16, + out_visual_dim=4, + patch_size=[1, 2, 2], + model_dim=32, + ff_dim=64, + num_text_blocks=0, + num_visual_blocks=2, + axes_dims=[8, 4, 4], + visual_cond=False, + instruct_type="noise", + use_text=False, + n_grid=3, + attribute_overrides={"instruct_type": "noise"}, +) + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29508", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +@pytest.fixture(autouse=True) +def cpu_only(monkeypatch): + """Load on CPU; the forward test moves its DiT to CUDA for fused kernels.""" + monkeypatch.setattr( + component_loader_module.ComponentLoader, + "target_device", + staticmethod(lambda component_starts_on_cpu: torch.device("cpu")), + ) + + +def _dit_config(**changes): + flat = dict(TINY_DIT, **changes) + config = Kandinsky6SRDitConfig() + config.update_model_arch(flat) + return config, flat + + +def _dit(**changes): + """Tiny DiT with deterministic random weights (the linears start uninitialized).""" + config, flat = _dit_config(**changes) + model = Kandinsky6SRTransformer3DModel(config, flat) + generator = torch.Generator().manual_seed(0) + with torch.no_grad(): + for param in model.parameters(): + param.copy_(torch.randn(param.shape, generator=generator) * 0.05) + return model + + +def _official_state_dict(model): + """State dict as the current Diffusers checkpoint stores it.""" + return { + name.replace(".feed_forward.mlp.fc_in.", ".feed_forward.net.0.proj.") + .replace(".feed_forward.mlp.fc_out.", ".feed_forward.net.2.") + .replace( + "time_embeddings.in_layer.", + "time_embeddings.timestep_embedder.linear_1.", + ) + .replace( + "time_embeddings.out_layer.", + "time_embeddings.timestep_embedder.linear_2.", + ): tensor + for name, tensor in model.state_dict().items() + } + + +def _load_through_the_real_loader(checkpoint, **changes): + """Meta-init the DiT and load ``checkpoint`` like ``maybe_load_fsdp_model`` does.""" + config, flat = _dit_config(**changes) + with torch.device("meta"): + model = Kandinsky6SRTransformer3DModel(config, flat) + load_model_from_full_model_state_dict( + model, + model.preprocess_loaded_state_dict(iter(checkpoint.items())), + torch.device("cpu"), + torch.float32, + strict=False, + param_names_mapping=get_param_names_mapping(model.param_names_mapping), + ) + return model + + +# --------------------------------------------------------------------------- # +# DiT +# --------------------------------------------------------------------------- # +def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt(): + """Positive control for the failure tests below, and the meta-device path: RoPE + tables and time-embedding frequencies are not in the checkpoint and must be rebuilt + by ``post_load_weights`` so that a loaded model actually runs.""" + reference = _dit() + checkpoint = _official_state_dict(reference) + model = _load_through_the_real_loader(checkpoint) + model.post_load_weights() + assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()]) + assert not model.time_embeddings.freqs.is_meta + torch.testing.assert_close(model.pooled_bias, reference.pooled_bias) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.to(device) + x = torch.randn(1, 2, 4, 4, 4, device=device) + rope_pos = [torch.arange(2, device=device) for _ in range(3)] + out = model(x, torch.tensor([500.0], device=device), rope_pos) + assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all() + + +@pytest.mark.parametrize( + "missing", + [ + "pooled_bias", + "visual_transformer_blocks.0.visual_modulation.out_layer.bias", + ], +) +def test_missing_checkpoint_tensor_fails_the_load(missing): + """The generic loader zero-fills missing parameters whose names contain ``bias`` (and + would only warn about others). For this model every parameter is required, so a + partial checkpoint must raise instead of yielding a silently different network.""" + checkpoint = _official_state_dict(_dit()) + del checkpoint[missing] + with pytest.raises(ValueError, match="was not loaded"): + _load_through_the_real_loader(checkpoint) + + +def test_unexpected_checkpoint_tensor_fails_the_load(): + checkpoint = _official_state_dict(_dit()) + checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2) + with pytest.raises(ValueError, match="unexpected keys"): + _load_through_the_real_loader(checkpoint) + + +def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys(): + """An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives + in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``. + """ + official = { + key: value + for key, value in TINY_DIT.items() + if key not in ("n_grid", "attribute_overrides") + } + official.update( + out_visual_dim=12, + attribute_overrides=None, + sr_params=dict( + scale_factor={"512": [1.0, 1.0, 1.0]}, + visual_size=[512], + scheduler_scale=5.0, + lq_noise_scale=0.7, + lq_noise_type="ddpm", + lq_channel_noise_scale=0.0, + cap_noise_timestep=False, + fps=24, + ), + ) + config = Kandinsky6SRDitConfig() + config.update_model_arch(official) + with torch.device("meta"): + model = Kandinsky6SRTransformer3DModel(config, official) + # head width = prod(patch_size) * the TOTAL out_visual_dim + assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12 + assert model.base_out_visual_dim == 4 and model.n_grid == 1 + + load_model_from_full_model_state_dict( + model, + model.preprocess_loaded_state_dict( + iter( + _official_state_dict(_dit()).items() + ) # same shapes as the legacy DX head + ), + torch.device("cpu"), + torch.float32, + strict=False, + param_names_mapping=get_param_names_mapping(model.param_names_mapping), + ) + model.post_load_weights() + assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()]) + + +def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise(): + """A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise`` + override would feed ``in`` channels. That mismatch must be reported, and the + documented ``visual_cond`` override must make it run.""" + wide = dict( + instruct_type="hybrid_anchor", attribute_overrides={"instruct_type": "noise"} + ) + with pytest.raises(ValueError, match="visual_cond"): + _dit(**wide) + + model = _dit( + instruct_type="hybrid_anchor", + attribute_overrides={"instruct_type": "noise", "visual_cond": True}, + ) + assert model.instruct_type == "noise" and model.visual_cond is True + assert model.visual_embed_dim == 9 + assert model.config.instruct_type == "hybrid_anchor" # the trained value is kept + + +def test_rope_angle_tables_stay_fp32_when_the_module_is_cast(): + """The residency manager casts a module to its target dtype (bf16) on every move to + the GPU; the RoPE angle tables must not be rounded to bf16 with the weights.""" + model = _dit().to(torch.bfloat16) + assert model.pooled_bias.dtype == torch.bfloat16 + for name, buffer in model.visual_rope_embeddings.named_buffers(): + assert buffer.dtype == torch.float32, name + + +# --------------------------------------------------------------------------- # +# Fake server args for the component loaders +# --------------------------------------------------------------------------- # +class FakeServerArgs: + def __init__(self): + self.pipeline_config = Kandinsky6SRPipelineConfig() + self.model_paths = {} + self.component_paths = {} + self.component_precisions = {} + self.component_weights_paths = {} + self.component_quantizations = {} + self.num_gpus = 1 + self.disable_autocast = False + self.revision = None + self.trust_remote_code = False + + def should_start_component_on_cpu(self, component_name): + return False + + def should_direct_gpu_weight_load_component(self, component_name): + return False + + def should_use_fsdp_for_component(self, component_name): + return False + + def requested_component_attention_backend(self, component_name): + return None + + def resolve_component_attention_backend(self, *names): + return None, None + + +def _write_component(directory, config, tensors): + directory.mkdir(parents=True, exist_ok=True) + (directory / "config.json").write_text(json.dumps(config)) + save_file(tensors, str(directory / "diffusion_pytorch_model.safetensors")) + + +def _tiny_vae_bundle(tmp_path): + raw_config = dict( + _class_name="Kandinsky6SRVAE", + vae_type="video-kvae", + encoder_config=dict(TINY_KVAE, in_channels=3), + decoder_config=dict(TINY_KVAE, out_ch=3), + scaling_factor=0.5, + spatial_factor=16, + temporal_factor=4, + ) + config = Kandinsky6SRVAEConfig() + config.update_model_arch(raw_config) + wrapper = Kandinsky6SRVAE(config) + directory = tmp_path / "vae" + _write_component( + directory, + raw_config, + {k: v.contiguous() for k, v in wrapper.state_dict().items()}, + ) + return directory + + +def test_vae_loader_loads_the_wrapper_strictly(tmp_path): + directory = _tiny_vae_bundle(tmp_path) + server_args = FakeServerArgs() + vae = VAELoader().load_customized(str(directory), server_args, "vae") + assert isinstance(vae, Kandinsky6SRVAE) and vae.scaling_factor == 0.5 + assert next(vae.parameters()).dtype == torch.bfloat16 # vae_precision + assert (vae.spatial_factor, vae.temporal_factor) == (16, 4) + + # one missing tensor must fail: non-strict VAE loading would only log a warning + weights_path = directory / "diffusion_pytorch_model.safetensors" + weights = load_file(str(weights_path)) + torch.testing.assert_close( + vae.state_dict(), + {name: value.bfloat16() for name, value in weights.items()}, + rtol=0, + atol=0, + ) + weights.pop(next(iter(weights))) + save_file(weights, str(weights_path)) + with pytest.raises(RuntimeError, match="Missing key"): + VAELoader().load_customized(str(directory), FakeServerArgs(), "vae") + + +def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True): + models = models or [ + {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} + for scale in ("2x", "4x") + ] + bank = Kandinsky6SRLatentUpscalerBank(models, 0.5, scales=scales) + directory = tmp_path / "latent_upscaler" + config = { + "_class_name": "Kandinsky6SRLatentUpscalerBank", + "models": models, + "scaling_factor": 0.5, + } + if include_scales: + config["scales"] = list(scales) + _write_component( + directory, + config, + {k: v.contiguous() for k, v in bank.state_dict().items()}, + ) + return directory + + +def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path): + directory = _tiny_lu_bundle(tmp_path) + bank, _ = LatentUpscalerLoader().load( + str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" + ) + assert bank.scales == (2, 4) + assert not bank.training + assert all( + p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters() + ) + z = torch.randn(1, 4, 3, 4, 6, dtype=torch.bfloat16) + assert bank.upscale(z, scale=2).shape == (1, 4, 3, 8, 12) + assert bank.upscale(z, scale=4).shape == (1, 4, 3, 16, 24) + with pytest.raises(ValueError, match="no x8 entry"): + bank.upscale(z, scale=8) + + +def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path): + models = [ + {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} + for scale in ("4x", "2x") + ] + directory = _tiny_lu_bundle( + tmp_path, + models=models, + include_scales=False, + ) + bank, _ = LatentUpscalerLoader().load( + str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" + ) + assert bank.scales == (2, 4) + assert bank._models[0] is bank.for_scale(2) + assert bank._models[1] is bank.for_scale(4) + + +def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library(): + """The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...].""" + for library in ("diffusers", "kandinsky6"): + loader = component_loader_module.ComponentLoader.for_component_type( + "latent_upscaler", library + ) + assert isinstance(loader, LatentUpscalerLoader) + with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"): + component_loader_module.ComponentLoader.for_component_type( + "latent_upscaler", "transformers" + ) + + +def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path): + """``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the + customized loader raises, which hides real errors. This component must re-raise, + with the offending config key or tensor named.""" + directory = _tiny_lu_bundle(tmp_path) + weights_path = directory / "diffusion_pytorch_model.safetensors" + weights = load_file(str(weights_path)) + weights.pop(next(iter(weights))) + save_file(weights, str(weights_path)) + with pytest.raises(RuntimeError, match="native fallback is disabled") as error: + LatentUpscalerLoader().load( + str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" + ) + assert "Missing key" in str(error.value.__cause__) + + bad = copy.deepcopy(TINY_LU_MODEL) + bad["bogus_field"] = 1 + directory = _tiny_lu_bundle( + tmp_path / "bad", + models=[{"target_scale": "2x", "model": TINY_LU_MODEL}], + scales=(2,), + ) + config_path = directory / "config.json" + config = json.loads(config_path.read_text()) + config["models"][0]["model"] = bad + config_path.write_text(json.dumps(config)) + with pytest.raises(RuntimeError, match="native fallback is disabled") as error: + LatentUpscalerLoader().load( + str(directory), FakeServerArgs(), "latent_upscaler", "diffusers" + ) + assert "bogus_field" in str(error.value.__cause__) + + +def test_motion_attention_is_not_a_supported_entry_key(): + """The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` / + commit 950a5edb) only implements the released checkpoint, which never trains + natten/shifted-window motion attention: a config that asks for it must fail loudly + at construction instead of silently building an unsupported backend.""" + with_motion = { + **copy.deepcopy(TINY_LU_MODEL), + "enable_x2_entry": False, + "motion_attention": { + "spatial_kernel_size": 3, + "temporal_offsets": [-1, 1], + "after_mid_blocks": [1], + "num_heads": 2, + }, + } + with pytest.raises(ValueError, match="motion_attention"): + Kandinsky6SRLatentUpscalerBank( + [{"target_scale": "4x", "model": with_motion}], 0.5 + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py new file mode 100644 index 0000000000..75cff0afff --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_official_release_configs.py @@ -0,0 +1,369 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Validate release config shapes and component loading without downloading weights.""" + +import copy +import json +import os +from collections import Counter +from types import SimpleNamespace +from typing import NamedTuple + +import pytest +import torch +from kandinsky6_sr_release_configs import ( + DISTILLED_MODEL_INDEX, + DISTILLED_SCHEDULER_CONFIG, + DISTILLED_TRANSFORMER_CONFIG, + FLOW_MODEL_INDEX, + FLOW_SCHEDULER_CONFIG, + FLOW_SR_CONFIG, + FLOW_TRANSFORMER_CONFIG, + LATENT_UPSCALER_CONFIG, + VAE_CONFIG, +) + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + PipelineComponentLoader, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import ( + SchedulerLoader, +) +from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( + Kandinsky6SRTransformer3DModel, +) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import ( + Kandinsky6SRPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + build_sampling_spec, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + PiflowParams, +) + + +class Bundle(NamedTuple): + """One official repo: its configs and what they must resolve to.""" + + model_index: dict + transformer: dict + scheduler: dict + sr_config: dict | None + scheduler_cls: type + head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width + n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid`` + scheduler_scale: float # ``sr_params.scheduler_scale`` + calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5 + + +FLOW = Bundle( + model_index=FLOW_MODEL_INDEX, + transformer=FLOW_TRANSFORMER_CONFIG, + scheduler=FLOW_SCHEDULER_CONFIG, + sr_config=FLOW_SR_CONFIG, + scheduler_cls=FlowMatchEulerDiscreteScheduler, + head_width=64, + n_grid=1, + scheduler_scale=5.0, + calls_per_tile=5, +) +DISTILLED = Bundle( + model_index=DISTILLED_MODEL_INDEX, + transformer=DISTILLED_TRANSFORMER_CONFIG, + scheduler=DISTILLED_SCHEDULER_CONFIG, + sr_config=None, + scheduler_cls=PiflowScheduler, + head_width=640, + n_grid=10, + scheduler_scale=3.5, + calls_per_tile=2, +) +both_repos = pytest.mark.parametrize( + "bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"] +) + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29509", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def _prefixes(keys, depth=2): + return Counter(".".join(key.split(".")[:depth]) for key in keys) + + +def _dit_config(bundle): + config = Kandinsky6SRDitConfig() + config.update_model_arch(copy.deepcopy(bundle.transformer)) + return config + + +def _load_scheduler(bundle, tmp_path): + """The scheduler component through SGLang's own loader (a ``scheduler/`` dir).""" + directory = tmp_path / "scheduler" + directory.mkdir() + (directory / "scheduler_config.json").write_text(json.dumps(bundle.scheduler)) + server_args = SimpleNamespace(pipeline_config=Kandinsky6SRPipelineConfig()) + return SchedulerLoader().load_customized(str(directory), server_args) + + +@both_repos +def test_real_transformer_config_builds_the_release_dit(bundle): + config = _dit_config(bundle) + with torch.device("meta"): + model = Kandinsky6SRTransformer3DModel( + config, copy.deepcopy(bundle.transformer) + ) + state = model.state_dict() + per_block = Counter( + key.split(".")[1] + for key in state + if key.startswith("visual_transformer_blocks.") + ) + assert ( + len(state) == 459 and len(per_block) == 32 and set(per_block.values()) == {14} + ) + non_block = [k for k in state if not k.startswith("visual_transformer_blocks.")] + assert len(non_block) == 11 + # ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its + # n_grid in the scheduler config + assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792) + assert model.base_out_visual_dim == 64 and model.n_grid == 1 + assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width + assert model.instruct_type == "hybrid_anchor" and model.visual_cond + assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input + arch = config.arch_config + assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]} + assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow + + +@both_repos +def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path): + scheduler = _load_scheduler(bundle, tmp_path) + # PiflowScheduler subclasses the flow-matching scheduler: compare exact types + assert type(scheduler) is bundle.scheduler_cls + assert scheduler.config["shift"] == bundle.scheduler["shift"] + if bundle is DISTILLED: + assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10) + + +@both_repos +def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path): + """The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever + ``num_inference_steps`` is), the flow-matching repo to flow-Euler + (``num_inference_steps`` calls per tile directly -- this repo's own convention, not + the upstream Diffusers pipeline's timestep-grid-point count). + """ + scheduler = _load_scheduler(bundle, tmp_path) + arch = _dit_config(bundle).arch_config + + def spec(num_steps): + return build_sampling_spec( + arch=arch, + tiling_scale=2, + seed=42, + num_steps=num_steps, + tiles_batch_size=1, + tile_min_overlap=0.2, + scheduler=scheduler, + ) + + assert spec(5).steps_per_chunk == bundle.calls_per_tile + assert spec(5).scheduler_scale == bundle.scheduler_scale + if bundle is DISTILLED: + assert spec(5).piflow == PiflowParams( + nfe=2, + num_policy_substeps=128, + final_step_size_scale=0.5, + shift=3.5, + n_grid=10, + eps=1e-6, + ) + assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2 + else: + assert spec(5).piflow is None + assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9) + + +def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys(): + config = Kandinsky6SRVAEConfig() + config.update_model_arch(dict(VAE_CONFIG)) + with torch.device("meta"): + vae = Kandinsky6SRVAE(config) + # `self.encoder` / `self.decoder` are direct attributes of the wrapper (no intermediate + # `model.*` nesting), so its state_dict keys already are the official checkpoint's own + # `encoder.*` / `decoder.*` keys -- no stripping needed. + official = dict(vae.state_dict()) + assert len(official) == 378 + assert _prefixes(official) == Counter( + { + "decoder.up": 238, + "encoder.down": 86, + "decoder.mid": 28, + "encoder.mid": 12, + "decoder.norm_out": 5, + "decoder.conv_in": 2, + "decoder.conv_out": 2, + "encoder.conv_in": 2, + "encoder.conv_out": 2, + "encoder.norm_out": 1, + } + ) + assert vae.scaling_factor == pytest.approx(0.910344004631042) + assert vae.spatial_factor == 16 + # the official keys load strictly, with no remapping + vae.load_state_dict(official, strict=True, assign=True) + + +def test_real_latent_upscaler_config_builds_the_release_bank(): + with torch.device("meta"): + bank = Kandinsky6SRLatentUpscalerBank( + LATENT_UPSCALER_CONFIG["models"], + LATENT_UPSCALER_CONFIG["scaling_factor"], + scales=(2, 4), + ) + state = bank.state_dict() + assert len(state) == 489 + assert _prefixes(state) == Counter({"_models.0": 311, "_models.1": 178}) + x2 = _prefixes((key.split(".", 2)[2] for key in state if "_models.0." in key)) + assert x2["x2_branch.blocks"] == 44 and x2["x2_branch.adapter"] == 28 + assert bank.scales == (2, 4) + + +@both_repos +@pytest.mark.parametrize("upscaler", ["present", "missing", "null"]) +def test_real_model_index_is_accepted_by_the_module_loading_path( + bundle, tmp_path, monkeypatch, upscaler +): + """``ComposedPipelineBase.load_modules`` sees a model_index.json whose extra + ``_kandinsky6_sr`` entry is a dict (not a ``[library, class]`` pair), next to a root + ``sr_config.json``: only the four components of the pipeline may be requested, with + the scheduler class of the repo.""" + model_index = dict(bundle.model_index) + if upscaler == "missing": + model_index.pop("latent_upscaler") + elif upscaler == "null": + model_index["latent_upscaler"] = None + (tmp_path / "model_index.json").write_text(json.dumps(model_index)) + if bundle.sr_config is not None: + (tmp_path / "sr_config.json").write_text(json.dumps(bundle.sr_config)) + components = ["transformer", "vae", "scheduler"] + if upscaler == "present": + components.append("latent_upscaler") + for name in components: + (tmp_path / name).mkdir() + if name != "scheduler": # the scheduler is config-only + (tmp_path / name / "diffusion_pytorch_model.safetensors").touch() + + requested = {} + + def record_component( + component_name, + component_model_path, + transformers_or_diffusers, + server_args, + component_architecture=None, + component_type=None, + loader_cls=None, + component_attn_backend=None, + component_attn_name=None, + ): + assert component_type == component_name + assert loader_cls is None and component_attn_backend is None + assert component_attn_name == component_name + requested[component_name] = (transformers_or_diffusers, component_architecture) + return SimpleNamespace(name=component_name), 0.0 + + monkeypatch.setattr( + PipelineComponentLoader, "load_component", staticmethod(record_component) + ) + server_args = SimpleNamespace( + model_subfolder=None, + revision=None, + comfyui_mode=False, + component_paths={}, + component_direct_gpu_weight_loading={}, + pipeline_config=Kandinsky6SRPipelineConfig(), + resolve_component_attention_backend=lambda *names: (None, None), + ) + # the state ``ComposedPipelineBase.__init__`` sets up before it calls load_modules + pipeline = object.__new__(Kandinsky6SRPipeline) + pipeline.server_args = server_args + pipeline.model_path = str(tmp_path) + pipeline._required_config_modules = list( + Kandinsky6SRPipeline._required_config_modules + ) + pipeline._extra_config_module_map = {} + pipeline._disagg_role = RoleType.MONOLITHIC + pipeline.memory_usages = {} + + modules = pipeline.load_modules(server_args) + + expected = { + "transformer": ("diffusers", "Kandinsky6SRTransformer3DModel"), + "vae": ("diffusers", "Kandinsky6SRVAE"), + "latent_upscaler": ("diffusers", "Kandinsky6SRLatentUpscalerBank"), + "scheduler": ("diffusers", bundle.scheduler_cls.__name__), + } + if upscaler != "present": + expected.pop("latent_upscaler") + assert set(modules) == set(expected) + assert requested == expected + assert "latent_upscaler" in Kandinsky6SRPipeline._required_config_modules + + +def test_legacy_bundle_requires_official_diffusers_layout(tmp_path): + (tmp_path / "model_index.json").write_text( + json.dumps( + { + "_class_name": "Kandinsky6SRPipeline", + "_diffusers_version": "0.37.0", + "dit": ["x", "Kandinsky6SRDiT"], + "vae": ["x", "Kandinsky6SRVAE"], + } + ) + ) + for name in ("dit", "vae"): + (tmp_path / name).mkdir() + (tmp_path / name / "diffusion_pytorch_model.safetensors").touch() + pipeline = object.__new__(Kandinsky6SRPipeline) + pipeline.server_args = SimpleNamespace(model_subfolder=None, revision=None) + pipeline.model_path = str(tmp_path) + with pytest.raises( + ValueError, match="k6_video SR bundle.*kandinskylab/Kandinsky-6.0-VSR" + ): + pipeline.load_modules(pipeline.server_args) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py new file mode 100644 index 0000000000..76e3244c12 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py @@ -0,0 +1,829 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CPU parity of the Kandinsky 6 SR port against the k6_video reference (``kandinsky_sr``). + +The reference package is not part of this repository. Point +``KANDINSKY_SR_REFERENCE_SRC`` at the directory that contains ``kandinsky_sr`` (for +example ``k6_video/src``); without it the whole module is skipped. The reference needs +``flash_attn`` and ``loguru``, which are stubbed here (per-sequence SDPA for the dense +varlen attention), and its base resolutions are shrunk so that tiles stay tiny. All +components use tiny random weights (every parameter re-randomized: the reference +zero-initializes most modulation / output layers, which would hide bugs) and float32 +math on CPU. The port loads the reference ``state_dict()`` (unprefixed, i.e. exactly the official +Diffusers layout) with ``strict=True`` through its own name mapping. +""" + +import copy +import importlib.machinery +import os +import re +import sys +import types +from functools import partial +from types import SimpleNamespace + +import pytest +import torch +from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( + Kandinsky6SRVAEConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.loader.utils import ( + get_param_names_mapping, + hf_to_custom_state_dict, +) +from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( + Kandinsky6SRTransformer3DModel, +) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr import ( + tiling, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + build_dit_spec, + build_sampling_spec, + effective_scheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + denoise_with_scheduler, + euler_start_timestep, + make_dit_fn, + module_dtype, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + plan_tiles, + stitch_tiles, +) + +REFERENCE_ENV = "KANDINSKY_SR_REFERENCE_SRC" +_reference_dir = os.environ.get(REFERENCE_ENV) +if not _reference_dir or not os.path.isdir( + os.path.join(_reference_dir, "kandinsky_sr") +): + pytest.skip( + f"set {REFERENCE_ENV} to the directory containing the kandinsky_sr package", + allow_module_level=True, + ) + +# --------------------------------------------------------------------------- # +# Reference harness (test-only): stubs, tiny resolutions, tiny random components +# --------------------------------------------------------------------------- # +TINY_RESOLUTIONS = {512: [(64, 64), (64, 96), (96, 64)]} +TINY_KVAE_ENC = dict(TINY_KVAE, in_channels=3) +TINY_KVAE_DEC = dict(TINY_KVAE, out_ch=3) +TINY_SCALING_FACTOR = 0.5 +TINY_DIT_CFG = dict( + in_visual_dim=4, + in_text_dim=8, + in_text_dim2=8, + time_dim=16, + out_visual_dim=4, + patch_size=(1, 2, 2), + model_dim=32, + ff_dim=64, + num_text_blocks=0, + num_visual_blocks=2, + axes_dims=(8, 4, 4), + visual_cond=False, + instruct_type="noise", + attention_params={"512": {"type": "flash"}}, + use_text=False, +) +TINY_PIFLOW = dict( + nfe=2, + num_policy_substeps=8, + final_step_size_scale=0.5, + shift=5.0, + n_grid=3, + eps=1e-6, +) +TINY_SR_PARAMS = dict( + scale_factor={512: [1.0, 1.0, 1.0]}, + visual_size=[512], + scheduler_scale=5.0, + lq_noise_scale=0.7, + lq_noise_type="ddpm", + lq_channel_noise_scale=0.0, + cap_noise_timestep=False, + fps=24, +) + + +def _varlen_sdpa(q, k, v, cu_q, cu_k): + outs = [] + for i in range(len(cu_q) - 1): + qs, qe = int(cu_q[i]), int(cu_q[i + 1]) + ks, ke = int(cu_k[i]), int(cu_k[i + 1]) + qi = q[qs:qe].transpose(0, 1).unsqueeze(0) + ki = k[ks:ke].transpose(0, 1).unsqueeze(0) + vi = v[ks:ke].transpose(0, 1).unsqueeze(0) + out = torch.nn.functional.scaled_dot_product_attention(qi, ki, vi) + outs.append(out.squeeze(0).transpose(0, 1)) + return torch.cat(outs, dim=0) + + +def _flash_attn_stub() -> types.ModuleType: + module = types.ModuleType("flash_attn") + + def varlen( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + return_attn_probs=False, + **_, + ): + out = _varlen_sdpa(q, k, v, cu_seqlens_q, cu_seqlens_k) + return (out, torch.zeros(1), None) if return_attn_probs else out + + def qkvpacked(qkv, cu_seqlens, max_seqlen, return_attn_probs=False, **_): + q, k, v = qkv.unbind(dim=1) + out = _varlen_sdpa(q, k, v, cu_seqlens, cu_seqlens) + return (out, torch.zeros(1), None) if return_attn_probs else out + + module.flash_attn_varlen_func = varlen + module.flash_attn_varlen_qkvpacked_func = qkvpacked + # transformers probes flash_attn with importlib.util.find_spec + module.__spec__ = importlib.machinery.ModuleSpec("flash_attn", loader=None) + return module + + +def _loguru_stub() -> types.ModuleType: + module = types.ModuleType("loguru") + + class _Logger: + def __getattr__(self, name): + return lambda *args, **kwargs: self + + module.logger = _Logger() + module.__spec__ = importlib.machinery.ModuleSpec("loguru", loader=None) + return module + + +def _install_reference() -> None: + os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") + os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") + if _reference_dir not in sys.path: + sys.path.insert(0, _reference_dir) + if "flash_attn" not in sys.modules: + sys.modules["flash_attn"] = _flash_attn_stub() + if "loguru" not in sys.modules: + sys.modules["loguru"] = _loguru_stub() + from kandinsky_sr import constants + + constants.RESOLUTIONS.clear() + constants.RESOLUTIONS.update(TINY_RESOLUTIONS) + + +_install_reference() + +from kandinsky_sr.core.components.model.dit import get_dit # noqa: E402 +from kandinsky_sr.core.components.model.dx_dit import DXDiTWrapper # noqa: E402 +from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402 + CachedCausalVAE, +) +from omegaconf import OmegaConf # noqa: E402 + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + """The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group.""" + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29507", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None: + generator = torch.Generator().manual_seed(seed) + with torch.no_grad(): + for param in module.parameters(): + param.copy_(torch.randn(param.shape, generator=generator) * std) + + +def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None = None): + cfg = dict(cfg or TINY_DIT_CFG) + if piflow is not None: + dit = DXDiTWrapper( + cfg, out_visual_dim=cfg["out_visual_dim"], n_grid=piflow["n_grid"] + ) + dit.piflow_params = dict(piflow) + else: + dit = get_dit(cfg) + dit.eval() + randomize(dit, seed) + return dit + + +def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict: + """The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields).""" + flat = dict(cfg) + flat["patch_size"] = list(flat["patch_size"]) + flat["axes_dims"] = list(flat["axes_dims"]) + flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1 + if piflow is not None: + flat.update( + piflow_nfe=piflow["nfe"], + piflow_num_policy_substeps=piflow["num_policy_substeps"], + piflow_final_step_size_scale=piflow["final_step_size_scale"], + piflow_shift=piflow["shift"], + piflow_eps=piflow["eps"], + ) + flat["attribute_overrides"] = dict(overrides) + flat.update( + sr_visual_size=[512], + sr_scale_factor={"512": [1.0, 1.0, 1.0]}, + sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"], + sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"], + sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"], + sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"], + sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"], + sr_fps=TINY_SR_PARAMS["fps"], + ) + return flat + + +def build_port_dit( + reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict +) -> Kandinsky6SRTransformer3DModel: + """Port DiT loaded with the reference weights (``model.`` prefix), strictly.""" + flat = flat_transformer_config( + cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides + ) + dit_config = Kandinsky6SRDitConfig() + dit_config.update_model_arch(flat) + model = Kandinsky6SRTransformer3DModel(dit_config, flat) + official = dict(reference_dit.state_dict()) # unprefixed = the official layout + mapped, _ = hf_to_custom_state_dict( + official, + get_param_names_mapping(model.param_names_mapping), + valid_target_names=set(model.state_dict()), + ) + model.load_state_dict(mapped, strict=True) + return model.eval() + + +# --------------------------------------------------------------------------- # +# DiT forward parity +# --------------------------------------------------------------------------- # +def _reference_forward(dit, x, time, *, scale_factor, motion_score=None): + """Run the packed reference DiT on a batched ``[B, T, H, W, C]`` input.""" + batch, frames, height, width, channels = x.shape + packed = x.reshape(batch * frames, height, width, channels) + visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32) + text_cu = torch.zeros(batch + 1, dtype=torch.int32) + rope_pos = [ + torch.cat([torch.arange(frames) for _ in range(batch)]), + torch.arange(height // dit.patch_size[1]), + torch.arange(width // dit.patch_size[2]), + ] + out = dit( + packed, + torch.zeros(0, 1), + torch.zeros(batch, 1), + time, + visual_cu, + text_cu, + rope_pos, + torch.zeros(0, dtype=torch.long), + scale_factor=scale_factor, + motion_score=motion_score, + ) + return out.reshape(batch, frames, *out.shape[1:]) if out.dim() == 4 else out + + +def _port_forward(model, x, time, *, scale_factor, motion_score=None): + batch, frames, height, width, _ = x.shape + rope_pos = [ + torch.arange(frames), + torch.arange(height // model.patch_size[1]), + torch.arange(width // model.patch_size[2]), + ] + return model( + x, + time, + rope_pos, + scale_factor=scale_factor, + motion_score=motion_score, + ) + + +def _random_latent( + width_channels: int, *, batch=2, frames=3, height=4, width=6, seed=0 +): + generator = torch.Generator().manual_seed(seed) + return torch.randn( + batch, frames, height, width, width_channels, generator=generator + ) + + +@pytest.mark.parametrize("piflow", [TINY_PIFLOW, None], ids=["dx_head", "plain_head"]) +def test_dit_forward_matches_reference(piflow): + """Guards the block math, RoPE, modulation, time bias and the DX head layout. + + The port is batched ``[B, T, H, W, C]`` while the reference packs samples with + ``cu_seqlens``; different per-sample times make a wrong batching/broadcast visible. + """ + reference = build_reference_dit(piflow=piflow) + model = build_port_dit( + reference, piflow=piflow, overrides={"instruct_type": "noise"} + ) + x = _random_latent(4) + time = torch.tensor([300.0, 850.0]) + scale = (1.0, 2.0, 2.0) + with torch.no_grad(): + expected = _reference_forward(reference, x, time, scale_factor=scale) + actual = _port_forward(model, x, time, scale_factor=scale) + n_grid = 1 if piflow is None else piflow["n_grid"] + if piflow is not None: + # reference DX wrapper already splits the grids: [B*T, n_grid, H, W, C] + expected = expected.movedim(1, -2).reshape(*actual.shape) + assert actual.shape[-1] == 4 * n_grid + torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) + + +def test_dit_forward_matches_reference_with_motion_score_and_wide_input(): + """Guards the optional motion embedding and the ``2 * in + 1`` conditioned input.""" + cfg = dict( + TINY_DIT_CFG, + use_motion_score=True, + instruct_type="hybrid_anchor", + use_lq_noise_cond=True, + ) + reference = build_reference_dit(piflow=TINY_PIFLOW, cfg=cfg) + model = build_port_dit( + reference, + cfg=cfg, + piflow=TINY_PIFLOW, + overrides={"instruct_type": "noise", "visual_cond": True}, + ) + reference.instruct_type = "noise" + reference.visual_cond = True + x = _random_latent(2 * 4 + 1) + time = torch.tensor([120.0, 999.0]) + motion = torch.full((1,), 900.0) + with torch.no_grad(): + expected = _reference_forward( + reference, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion + ) + actual = _port_forward( + model, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion + ) + without_motion = _port_forward(model, x, time, scale_factor=(1.0, 1.0, 1.0)) + expected = expected.movedim(1, -2).reshape(*actual.shape) + torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) + assert not torch.allclose(actual, without_motion) + + +def test_state_dict_keys_equal_mapped_reference_keys(): + """Guards the module tree / key mapping: extra or missing keys would break real checkpoints.""" + reference = build_reference_dit(piflow=TINY_PIFLOW) + model = build_port_dit(reference, piflow=TINY_PIFLOW, overrides={}) + mapped, _ = hf_to_custom_state_dict( + dict(reference.state_dict()), + get_param_names_mapping(model.param_names_mapping), + ) + assert set(mapped) == set(model.state_dict()) + + +# --------------------------------------------------------------------------- # +# Sampler loops (pi-Flow / Euler) against the reference generate functions +# --------------------------------------------------------------------------- # +def _reference_loop_inputs(x): + batch, frames, height, width, _ = x.shape + visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32) + text = {"text_embeds": torch.zeros(0, 1), "pooled_embed": torch.zeros(batch, 1)} + text_cu = torch.zeros(batch + 1, dtype=torch.int32) + rope_pos = [ + torch.cat([torch.arange(frames) for _ in range(batch)]), + torch.arange(height // 2), + torch.arange(width // 2), + ] + return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long) + + +def _port_dit_fn(model, x, scale_factor, use_motion_score=False): + return make_dit_fn( + model, + latent_frames_hw=tuple(x.shape[1:4]), + patch_size=model.patch_size, + scale_factor=scale_factor, + use_motion_score=use_motion_score, + ) + + +def test_piflow_loop_matches_reference_piflow_generate(): + """Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles. + + The reference packs the tiles along time with one per-frame sigma; the port batches + ``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU. + + The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule`` + / ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through + ``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the + ``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same + per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did. + """ + from kandinsky_sr.core.algo.piflow_sampler import piflow_generate + + reference = build_reference_dit(piflow=TINY_PIFLOW) + model = build_port_dit( + reference, piflow=TINY_PIFLOW, overrides={"instruct_type": "noise"} + ) + x = _random_latent(4, seed=5) + scale = (1.0, 2.0, 2.0) + visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) + packed = x.reshape(-1, *x.shape[2:]).clone() + scheduler = PiflowScheduler(**TINY_PIFLOW) + scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu") + with torch.no_grad(): + expected = piflow_generate( + packed, + reference, + text, + visual_cu, + text_cu, + rope_pos, + text_rope, + scale, + out_dim=4, + start_timestep=1.0, + device="cpu", + **TINY_PIFLOW, + ) + actual = denoise_with_scheduler( + x.clone(), + _port_dit_fn(model, x, scale), + scheduler, + channels=4, + is_piflow=True, + ) + torch.testing.assert_close( + actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 + ) + + +@pytest.mark.parametrize( + "capped", [False, True], ids=["full_range", "capped_noise_start"] +) +def test_euler_loop_matches_reference_generate(capped): + """Guards the warped timesteps, per-step deltas and (for a wide input) the in-place + update of the first channels only, incl. the ``cap_noise_timestep`` start time. + + The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4 + Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent + port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace + array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift`` + warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself -- + algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4 + step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for- + bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array. + """ + from kandinsky_sr.core.algo.utils import generate + + cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor") + reference = build_reference_dit(piflow=None, cfg=cfg) + model = build_port_dit( + reference, + cfg=cfg, + piflow=None, + overrides={"instruct_type": "noise", "visual_cond": True}, + ) + reference.instruct_type, reference.visual_cond = "noise", True + x = _random_latent(2 * 4 + 1, seed=9) + scale = (1.0, 1.5, 1.5) + start = euler_start_timestep( + cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise" + ) + assert start == (0.7 if capped else 1.0) + visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) + packed = x.reshape(-1, *x.shape[2:]).clone() + num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps + scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0) + sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist() + scheduler.set_timesteps(sigmas=sigmas, device="cpu") + with torch.no_grad(): + expected = generate( + packed, + reference, + "cpu", + 5, + text, + text, + visual_cu, + text_cu, + text_cu, + rope_pos, + text_rope, + text_rope, + scale, + 1.0, + 5.0, + start_timestep=start, + ) + actual = denoise_with_scheduler( + x.clone(), + _port_dit_fn(model, x, scale, use_motion_score=True), + scheduler, + channels=4, + is_piflow=False, + ) + torch.testing.assert_close( + actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 + ) + + +# --------------------------------------------------------------------------- # +# Tiling / stitching against the reference helpers +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "frame_hw,scale", [((64, 96), 2), ((64, 64), 4), ((96, 64), 2), ((80, 144), 2)] +) +def test_tile_plan_and_stitch_match_reference(frame_hw, scale): + """Guards the tile grid (aspect -> base resolution -> even grid) and that the + frame-chunked stitch equals one reference stitch over the whole clip (T=19, not a + multiple of the chunk size).""" + from kandinsky_sr.core.algo.tiling_utils import stitch_tiles_hanning + from kandinsky_sr.pipeline.sr_pipeline import _tile_geometry + + height, width = frame_hw + plan = plan_tiles( + frame_hw=frame_hw, + visual_size=512, + tiling_scale=scale, + tile_min_overlap=0.2, + resolutions=TINY_RESOLUTIONS, + ) + base, tile_hw, grid = _tile_geometry( + height, width, 512, scale, 0.25, 16, "even", 0.2 + ) + assert (plan.base_hw, plan.tile_hw) == (base, tile_hw) + assert tuple(plan.pixel_grid) == tuple(grid) + generator = torch.Generator().manual_seed(3) + tiles = [ + torch.randint(0, 256, (3, 19, *base), dtype=torch.uint8, generator=generator) + for _ in range(grid.total_tiles) + ] + expected = ( + stitch_tiles_hanning( + [t.float() for t in tiles], grid, height, width, scale=scale + ) + .clamp(0, 255) + .to(torch.uint8) + ) + assert torch.equal(stitch_tiles(tiles, plan), expected) + + +# --------------------------------------------------------------------------- # +# End to end: pure orchestration vs kandinsky_sr.pipeline.stages.run_tiled_sr* +# --------------------------------------------------------------------------- # +def build_reference_lu_bank(target_scales=("2x", "4x"), seed=0): + """Reference LU bank with random weights and its bundle-style ``models`` config.""" + import pydantic + from kandinsky_sr.core.algo.latent_upscaler import LatentUpscalerBank + from kandinsky_sr.core.components.latent_upscaler.config import ModelConfig + from kandinsky_sr.core.components.latent_upscaler.model.factory import ( + build_upsampler, + ) + + models = [ + {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} + for scale in target_scales + ] + adapter = pydantic.TypeAdapter(ModelConfig) + bank = LatentUpscalerBank() + for index, spec in enumerate(models): + upsampler = build_upsampler(adapter.validate_python(spec["model"])).eval() + randomize(upsampler, seed + 10 + index) + upsampler.target_scale = spec["target_scale"] + upsampler.scaling_factor = TINY_SCALING_FACTOR + bank[spec["target_scale"]] = upsampler + return bank, models + + +def build_stacks(*, piflow, use_lu): + """(reference SRComponents, port dict) sharing identical random weights.""" + from kandinsky_sr.pipeline.sr_pipeline import SRComponents, SRParams + + torch.manual_seed(0) + ref_vae = CachedCausalVAE( + encoder_conf=OmegaConf.create(dict(TINY_KVAE_ENC)), + decoder_conf=OmegaConf.create(dict(TINY_KVAE_DEC)), + ).eval() + randomize(ref_vae, 1, std=0.1) + ref_vae.config = SimpleNamespace(scaling_factor=TINY_SCALING_FACTOR) + ref_dit = build_reference_dit(piflow=piflow, seed=2) + ref_bank, models = build_reference_lu_bank() if use_lu else (None, None) + reference = SRComponents( + dit=ref_dit, + vae=ref_vae, + latent_upscaler=ref_bank, + sr_params=SRParams(**TINY_SR_PARAMS), + spatial_factor=16, + ) + + vae_config = Kandinsky6SRVAEConfig() + vae_config.update_model_arch( + dict( + vae_type="video-kvae", + encoder_config=dict(TINY_KVAE_ENC), + decoder_config=dict(TINY_KVAE_DEC), + scaling_factor=TINY_SCALING_FACTOR, + spatial_factor=16, + temporal_factor=4, + ) + ) + vae = Kandinsky6SRVAE(vae_config) + # unprefixed reference keys = the official layout, mapped by the wrapper's load hook + vae.load_state_dict(dict(ref_vae.state_dict()), strict=True) + port = { + "vae": vae.eval(), + "dit": build_port_dit( + ref_dit, piflow=piflow, overrides={"instruct_type": "noise"} + ), + "bank": None, + } + if use_lu: + bank = Kandinsky6SRLatentUpscalerBank(models, TINY_SCALING_FACTOR) + converted = {} + scale_indices = {"2x": 0, "4x": 1} + for key, value in ref_bank.state_dict().items(): + scale, suffix = key.split(".", 1) + suffix = suffix.replace("private_", "") + suffix = re.sub(r"(output_proj\.)0\.", r"\1norm.", suffix) + suffix = re.sub(r"(output_proj\.)2\.", r"\1conv.", suffix) + converted[f"_models.{scale_indices[scale]}.{suffix}"] = value + bank.load_state_dict( + converted, + strict=True, + ) + port["bank"] = bank.eval() + return reference, port + + +def run_reference(video, reference, *, scale, seed, tiles_batch_size, num_steps): + from kandinsky_sr.core.algo.latent_upscaler import latent_upscaler_for_scale + from kandinsky_sr.pipeline.sr_pipeline import ( + RunConfig, + encode_lq_video_to_lr_latent, + ) + from kandinsky_sr.pipeline.stages import run_tiled_sr, run_tiled_sr_from_pixels + from kandinsky_sr.pipeline.upscale_utils import ( + pre_upscale_video, + resolve_scale_request, + ) + + tiling_scale, pre = resolve_scale_request(float(scale)) + if pre != 1.0: + video = pre_upscale_video(video, pre, 16) + run_config = RunConfig( + device="cpu", + num_steps=num_steps, + seed=seed, + overlap=0.25, + tiles_batch_size=tiles_batch_size, + resolution_scale=tiling_scale, + ) + if latent_upscaler_for_scale(reference.latent_upscaler, tiling_scale) is not None: + lr_latent = encode_lq_video_to_lr_latent(video, reference.vae, "cpu") + return run_tiled_sr(lr_latent, reference, run_config) + return run_tiled_sr_from_pixels(video, reference, run_config) + + +def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps): + """The stage logic (scale request, pre-upscale, specs) around the pure function. + + ``num_steps`` here is the reference's convention (timestep grid points); for a + flow-matching (non pi-Flow) bundle the port's own ``num_inference_steps`` is + ``num_steps - 1`` (DiT calls), so it is translated once, here, at the one remaining + boundary between the two conventions. A pi-Flow bundle ignores it either way (its + scheduler's own ``nfe`` always wins), so no translation is needed for that case. + """ + tiling_scale, pre = tiling.resolve_scale_request(float(scale)) + if pre != 1.0: + video = tiling.pre_upscale_video(video, pre, 16) + arch = port["dit"].config + port_num_steps = num_steps if arch.is_piflow else num_steps - 1 + spec = build_sampling_spec( + arch=arch, + tiling_scale=tiling_scale, + seed=seed, + num_steps=port_num_steps, + tiles_batch_size=tiles_batch_size, + tile_min_overlap=0.2, + ) + bank = port["bank"] + use_lu = bank is not None and bank.for_scale(tiling_scale) is not None + return super_resolve( + video, + vae=port["vae"], + scaling_factor=port["vae"].scaling_factor, + dit=port["dit"], + dit_spec=build_dit_spec(port["dit"]), + spec=spec, + scheduler=effective_scheduler(spec, None), + device="cpu", + upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None, + lu_dtype=module_dtype(bank) if use_lu else None, + resolutions=TINY_RESOLUTIONS, + ) + + +def _random_video(frames, height, width, seed=0): + generator = torch.Generator().manual_seed(seed) + return torch.randint( + 0, 256, (frames, 3, height, width), dtype=torch.uint8, generator=generator + ) + + +E2E_CASES = [ + # scale, (H, W), tiles_batch_size + (2, (64, 96), 1), + (2, (64, 96), 3), + (4, (64, 64), 2), + (2.25, (64, 96), 1), +] + + +def _assert_uint8_close(actual, expected, *, max_diff: int = 1): + assert actual.shape == expected.shape and actual.dtype == expected.dtype + diff = (actual.int() - expected.int()).abs() + exact = (diff == 0).float().mean().item() + assert diff.max().item() <= max_diff and exact > 0.999, (diff.max().item(), exact) + + +@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"]) +@pytest.mark.parametrize("scale,hw,tiles_batch_size", E2E_CASES) +def test_tiled_piflow_super_resolution_matches_reference( + use_lu, scale, hw, tiles_batch_size +): + """Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or + tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with + truncation to uint8 and Hann stitching, against ``run_tiled_sr`` / + ``run_tiled_sr_from_pixels`` with identical random weights. + + ``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the + training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to + 0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``). + The port now deliberately matches the real ``PiflowScheduler._policy_step`` + convention (``raw_dst=eps``) instead, which real checkpoints always run through. + That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up + here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing + rounding-boundary noise (still >99.99% exact pixels), not as a structural bug. + """ + reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu) + video = _random_video(9, *hw) + kwargs = dict(scale=scale, seed=42, tiles_batch_size=tiles_batch_size, num_steps=5) + expected = run_reference(video, reference, **kwargs) + actual = run_port(video, port, **kwargs) + _assert_uint8_close(actual, expected, max_diff=2) + + +@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"]) +def test_tiled_euler_super_resolution_matches_reference(use_lu): + """Same as above for a plain (non pi-Flow) DiT: the Euler loop with ``num_steps``.""" + reference, port = build_stacks(piflow=None, use_lu=use_lu) + video = _random_video(9, 64, 96, seed=1) + kwargs = dict(scale=2, seed=7, tiles_batch_size=2, num_steps=4) + expected = run_reference(video, reference, **kwargs) + actual = run_port(video, port, **kwargs) + _assert_uint8_close(actual, expected) + + +def test_seed_and_chunking_change_the_result_like_the_reference(): + """The initial noise of chunk k is seeded with ``seed + chunk start``: another seed or + another chunk size must change the output (a port that ignored them would not).""" + _, port = build_stacks(piflow=TINY_PIFLOW, use_lu=False) + video = _random_video(9, 64, 96, seed=2) + base = run_port(video, port, scale=2, seed=42, tiles_batch_size=1, num_steps=5) + assert not torch.equal( + base, run_port(video, port, scale=2, seed=43, tiles_batch_size=1, num_steps=5) + ) + assert not torch.equal( + base, run_port(video, port, scale=2, seed=42, tiles_batch_size=4, num_steps=5) + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py new file mode 100644 index 0000000000..bf138031f0 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sampler_selection.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Validate scheduler selection, DiT call counts and incompatible head rejection.""" + +import os +from typing import NamedTuple + +import pytest +from kandinsky6_sr_release_configs import ( + DISTILLED_SCHEDULER_CONFIG, + DISTILLED_TRANSFORMER_CONFIG, + FLOW_SCHEDULER_CONFIG, + FLOW_TRANSFORMER_CONFIG, +) +from kandinsky6_sr_tiny_components import ( + TINY_DIT, + build_components, + make_request, + make_stage, + random_video, +) + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( + PiflowScheduler, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( + Kandinsky6SRDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( + Kandinsky6SRLatentPrepStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_VIDEO_KEY, + build_sampling_spec, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + RESOLUTIONS, +) + +LATENT_WIDTH = TINY_DIT["in_visual_dim"] +N_GRID = ( + 3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields +) +NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles + + +def official_dit(head_width): + """Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL + head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config.""" + arch = { + key: value + for key, value in TINY_DIT.items() + if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_")) + } + return dict( + arch, + out_visual_dim=head_width, + sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]), + ) + + +FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head +DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head + + +def flow_scheduler(): + return FlowMatchEulerDiscreteScheduler(shift=5.0) + + +def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID): + return PiflowScheduler( + nfe=nfe, + n_grid=n_grid, + shift=5.0, + eps=1e-6, + final_step_size_scale=0.5, + num_policy_substeps=8, + ) + + +def release_scheduler(scheduler_cls, config): + return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")}) + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29510", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +@pytest.fixture(autouse=True) +def tiny_resolutions(monkeypatch): + monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)]) + + +class SamplingRun(NamedTuple): + tiles: int + dit_calls: int + progress_totals: list + + +def run_denoising(dit_arch, scheduler, *, num_steps=5): + """Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count + the DiT calls next to the totals the denoising progress bar was opened with.""" + vae, dit, _, server_args = build_components((), dit_arch) + dit_calls = [] + dit.register_forward_hook(lambda module, args, output: dit_calls.append(1)) + latent_prep = make_stage( + Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler + ) + denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler) + progress_totals = [] + progress_bar = denoise.progress_bar + + def spy_on_progress_bar(**kwargs): + progress_totals.append(kwargs["total"]) + return progress_bar(**kwargs) + + denoise.progress_bar = spy_on_progress_bar + + batch = make_request(random_video(9, 64, 96), num_steps=num_steps) + batch = latent_prep.forward(batch, server_args) + tiles = len(batch.extra["kandinsky6_sr_chunks"]) + denoise.forward(batch, server_args) + return SamplingRun(tiles, len(dit_calls), progress_totals) + + +def sampling_spec(dit_arch, scheduler): + config = Kandinsky6SRDitConfig() + config.update_model_arch(dict(dit_arch)) + return build_sampling_spec( + arch=config.arch_config, + tiling_scale=2, + seed=42, + num_steps=5, + tiles_batch_size=1, + tile_min_overlap=0.2, + scheduler=scheduler, + ) + + +# --------------------------------------------------------------------------- # +# DiT calls per tile +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)]) +def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps( + nfe, num_steps +): + run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps) + assert run.tiles == NUM_TILES + assert run.dit_calls == NUM_TILES * nfe + assert run.progress_totals == [NUM_TILES * nfe] + + +@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)]) +def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile( + num_steps, calls_per_tile +): + run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps) + assert run.tiles == NUM_TILES + assert run.dit_calls == NUM_TILES * calls_per_tile + assert run.progress_totals == [NUM_TILES * calls_per_tile] + + +@pytest.mark.parametrize("num_steps", [3, 9]) +def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps): + """A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the + denoising stage synthesizes a matching PiflowScheduler from those arch fields + (run_spec.effective_scheduler) instead of leaving nothing to step.""" + run = run_denoising(TINY_DIT, None, num_steps=num_steps) + assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"] + assert run.progress_totals == [run.dit_calls] + + +# --------------------------------------------------------------------------- # +# Samplers that do not fit the DiT head +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize( + "scheduler", + [flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None], + ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"], +) +def test_dx_head_never_runs_flow_euler(scheduler): + with pytest.raises(ValueError, match="flow-Euler"): + sampling_spec(DX_DIT, scheduler) + + +@pytest.mark.parametrize( + "dit_arch,n_grid,needed", + [ + (FLOW_DIT, N_GRID, "3 x 4 = 12"), + (DX_DIT, 2, "2 x 4 = 8"), + (DX_DIT, 4, "4 x 4 = 16"), + ], + ids=["single_head", "too_few_grids", "too_many_grids"], +) +def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed): + with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"): + sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid)) + + +@pytest.mark.parametrize("nfe", [0, -1]) +def test_pi_flow_needs_at_least_one_call_per_tile(nfe): + with pytest.raises(ValueError, match="nfe >= 1"): + sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe)) + with pytest.raises(ValueError, match="nfe >= 1"): + sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None) + + +def test_release_transformer_and_scheduler_of_different_repos_are_rejected(): + """The real configs of the two official repos: their transformers differ only in + the head width, so one repo's transformer with the other's scheduler must fail.""" + flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG) + distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG) + + assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5 + assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2 + with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"): + sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow) + with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"): + sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled) + + +def test_a_mismatched_bundle_fails_before_any_tile_is_processed(): + vae, dit, _, server_args = build_components((), DX_DIT) + dit_calls = [] + dit.register_forward_hook(lambda module, args, output: dit_calls.append(1)) + latent_prep = make_stage( + Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler() + ) + batch = make_request(random_video(9, 64, 96)) + + with pytest.raises(ValueError, match="flow-Euler"): + latent_prep.forward(batch, server_args) + assert dit_calls == [] + assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py new file mode 100644 index 0000000000..23dae9b75c --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Verify SR token sharding, TP weight loading, and repeated distributed forwards.""" + +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( + Kandinsky6SRDitConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import ( + Kandinsky6SRPipelineConfig, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, +) +from sglang.multimodal_gen.runtime.layers.attention.selector import ( + global_force_attn_backend, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( + Kandinsky6SRTransformer3DModel, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.server_args import set_global_server_args +from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy +from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_DIT +from sglang.srt.utils.network import get_free_port_below_ephemeral + + +def _forward(rank, world_size, tp_size, ring, port, output_dir): + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + LOCAL_RANK=str(rank), + WORLD_SIZE=str(world_size), + ) + torch.cuda.set_device(rank) + set_mixed_precision_policy(torch.bfloat16, torch.float32) + global_force_attn_backend(AttentionBackendEnum.FA) + set_global_server_args( + SimpleNamespace( + attention_backend="fa", + attention_backend_config=None, + kv_gather_degree=1, + sp_split_auto=False, + pipeline_config=Kandinsky6SRPipelineConfig(), + ) + ) + sp_size = world_size // tp_size + maybe_init_distributed_environment_and_model_parallel( + tp_size=tp_size, + sp_size=sp_size, + ulysses_degree=1 if ring else sp_size, + ring_degree=sp_size if ring else 1, + ) + config = Kandinsky6SRDitConfig() + config.update_model_arch( + TINY_DIT | dict(model_dim=512, time_dim=32, ff_dim=1024, axes_dims=[32, 48, 48]) + ) + model = Kandinsky6SRTransformer3DModel(config, {}).eval() + generator = torch.Generator().manual_seed(7) + directory = Path(output_dir) + with torch.no_grad(): + if world_size == 1: + for parameter in model.parameters(): + parameter.copy_( + torch.randn(parameter.shape, generator=generator) * 0.02 + ) + torch.save(model.state_dict(), directory / "weights.pt") + else: + weights = torch.load(directory / "weights.pt", weights_only=True) + for name, parameter in model.named_parameters(): + loader = parameter.__dict__.get("weight_loader") + if loader is None: + parameter.copy_(weights[name]) + else: + loader(parameter, weights[name]) + model.to(device=rank, dtype=torch.bfloat16) + assert model.visual_transformer_blocks[0].self_attention.local_num_heads == ( + 4 // tp_size + ) + + observed = [] + + def record_shard(module, args): + visual, _, rope, _ = args + observed.append((visual.shape[0], visual.shape[1], rope.shape[1])) + + handle = model.visual_transformer_blocks[0].register_forward_pre_hook(record_shard) + first = None + for batch_size, frames in ((1, 2), (2, 3), (1, 2)): + generator.manual_seed(frames) + inputs = torch.randn(batch_size, frames, 6, 6, 4, generator=generator).to( + device=rank, dtype=torch.bfloat16 + ) + with ( + torch.no_grad(), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + actual = model( + inputs, + torch.full((batch_size,), 500.0, device=rank), + [torch.arange(n, device=rank) for n in (frames, 3, 3)], + ) + local_len = (frames * 3 * 3 + sp_size - 1) // sp_size + assert observed[-1] == (batch_size, local_len, local_len) + assert actual.shape == (batch_size, frames, 6, 6, 12) + assert torch.isfinite(actual).all() + if frames == 2: + if first is None: + first = actual.clone() + else: + torch.testing.assert_close(actual, first, rtol=0, atol=0) + path = directory / f"{batch_size}-{frames}.pt" + if world_size == 1: + torch.save(actual.cpu(), path) + else: + expected = torch.load(path, weights_only=True, map_location=f"cuda:{rank}") + torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002) + handle.remove() + dist.destroy_process_group() + + +def _launch(world_size, tp_size, ring, output_dir): + mp.spawn( + _forward, + args=(world_size, tp_size, ring, get_free_port_below_ephemeral(), output_dir), + nprocs=world_size, + ) + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs") +def test_sr_two_gpus(tmp_path): + _launch(1, 1, False, str(tmp_path)) + _launch(2, 1, False, str(tmp_path)) + _launch(2, 1, True, str(tmp_path)) + _launch(2, 2, False, str(tmp_path)) + + +@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs") +def test_sr_four_gpus(tmp_path): + _launch(1, 1, False, str(tmp_path)) + for tp_size in (1, 2, 4): + _launch(4, tp_size, False, str(tmp_path)) + _launch(4, 2, True, str(tmp_path)) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py new file mode 100644 index 0000000000..7e19ff0625 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_stages.py @@ -0,0 +1,353 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native SR stage-chain parity, component residency and video/audio hand-offs.""" + +import copy +import os +from functools import partial +from unittest.mock import MagicMock + +import pytest +import torch +from kandinsky6_sr_tiny_components import ( + TINY_LU_MODEL, + build_components, + make_request, + make_stage, + random_video, + super_resolve, +) + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( + Kandinsky6SRLatentUpscalerBank, +) +from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import ( + Kandinsky6SRPipeline, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import ( + Kandinsky6SRDecodeStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import ( + Kandinsky6SRDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import ( + Kandinsky6SREncodeStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( + WARMUP_CLIP_FRAMES, + WARMUP_CLIP_HW, + Kandinsky6SRInputStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import ( + Kandinsky6SRLatentPrepStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import ( + Kandinsky6SROutputStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_LR_LATENT_KEY, + SR_PLAN_KEY, + SR_REQUESTED_HW_KEY, + SR_TILES_KEY, + SR_VIDEO_KEY, + build_dit_spec, + build_sampling_spec, + effective_scheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + module_dtype, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + TilePlan, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + RESOLUTIONS, +) + + +@pytest.fixture(scope="module", autouse=True) +def single_process_model_parallel(): + if not model_parallel_is_initialized(): + for key, value in dict( + MASTER_ADDR="127.0.0.1", + MASTER_PORT="29509", + RANK="0", + LOCAL_RANK="0", + WORLD_SIZE="1", + ).items(): + os.environ.setdefault(key, value) + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +@pytest.fixture(autouse=True) +def tiny_resolutions(monkeypatch): + monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)]) + + +@pytest.mark.parametrize( + "bank_scales", + [("2x", "4x"), (), ("4x",)], + ids=["lu_path", "pixel_path", "pixel_fallback"], +) +def test_stage_chain_reproduces_the_pure_orchestration(bank_scales): + """encode -> latent-prep -> denoise -> decode -> output must give what + ``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank + that has no entry for the requested scale (which must fall back to the pixel path, not + fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and + 2.""" + vae, dit, bank, server_args = build_components(bank_scales) + video = random_video(9, 64, 128) + batch = make_request(video, tiles_batch_size=5) + + pipeline = object.__new__(Kandinsky6SRPipeline) + pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None) + if bank is not None: + pipeline.modules["latent_upscaler"] = bank + pipeline._stages, pipeline._stage_name_mapping = [], {} + pipeline._disagg_role = RoleType.MONOLITHIC + pipeline.create_pipeline_stages(server_args) + for stage in pipeline.stages: + stage.server_args = server_args + input_stage, encode, latent_prep, denoise, decode, output = pipeline.stages + assert [type(stage) for stage in pipeline.stages] == [ + Kandinsky6SRInputStage, + Kandinsky6SREncodeStage, + Kandinsky6SRLatentPrepStage, + Kandinsky6SRDenoisingStage, + Kandinsky6SRDecodeStage, + Kandinsky6SROutputStage, + ] + assert [ + (u.component_name, u.start_at_stage_entry) + for u in encode.component_uses(server_args) + ] == [("vae", False)] + expected_phases = [("vae", "encode_tiles")] + if bank is not None: + expected_phases.insert(0, ("latent_upscaler", "upscale_tiles")) + assert [ + (u.component_name, u.phase) for u in latent_prep.component_uses(server_args) + ] == expected_phases + (dit_use,) = denoise.component_uses(server_args) + assert (dit_use.component_name, dit_use.phase) == ("transformer", "denoise_tiles") + assert dit_use.memory_intensive and dit_use.target_dtype == torch.bfloat16 + assert [ + (u.component_name, u.phase) for u in decode.component_uses(server_args) + ] == [("vae", "decode_tiles")] + + batch = encode.forward(batch, server_args) + assert (SR_LR_LATENT_KEY in batch.extra) == ("2x" in bank_scales) + batch = latent_prep.forward(batch, server_args) + assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra + assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk + assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan) + batch = denoise.forward(batch, server_args) + batch = decode.forward(batch, server_args) + assert len(batch.extra[SR_TILES_KEY]) == 12 + result = output.forward(batch, server_args) + + arch = dit.config + use_lu = "2x" in bank_scales + spec = build_sampling_spec( + arch=arch, + tiling_scale=2, + seed=42, + num_steps=5, + tiles_batch_size=5, + tile_min_overlap=0.2, + ) + expected = super_resolve( + video, + vae=vae, + scaling_factor=vae.scaling_factor, + dit=dit, + dit_spec=build_dit_spec(dit), + spec=spec, + scheduler=effective_scheduler(spec, None), + device=next(dit.parameters()).device, + upscale_fn=partial(bank.upscale, scale=2) if use_lu else None, + lu_dtype=module_dtype(bank) if use_lu else None, + ) + assert isinstance(result, OutputBatch) + assert result.output.dtype == torch.float16 + assert result.output.shape == (1, 3, 9, 128, 256) + restored = (result.output[0].float() * 255).to(torch.uint8) + assert torch.equal(restored, expected) + assert (batch.height, batch.width) == (128, 256) + assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra + + +@pytest.mark.parametrize( + "role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER] +) +def test_sr_rejects_disaggregation(role): + pipeline = object.__new__(Kandinsky6SRPipeline) + with pytest.raises(ValueError, match="monolithic"): + pipeline.validate_disagg_role(role) + pipeline.validate_disagg_role(RoleType.MONOLITHIC) + + +def test_encode_stage_leaves_the_vae_alone_on_the_pixel_path(): + """No LU entry for the scale -> the whole-video encode is skipped, and the VAE (which + a CPU-offloaded run would otherwise move to the GPU for nothing) is never touched. + """ + vae = MagicMock(name="vae") + bank = Kandinsky6SRLatentUpscalerBank( + [{"target_scale": "4x", "model": copy.deepcopy(TINY_LU_MODEL)}], + 0.5, + scales=(4,), + ) + _, _, _, server_args = build_components(()) + video = random_video(9, 64, 64) + batch = make_request(video) + stage = make_stage(Kandinsky6SREncodeStage, server_args, vae, bank) + assert stage.forward(batch, server_args) is batch + assert torch.equal(batch.extra[SR_VIDEO_KEY], video) + assert SR_LR_LATENT_KEY not in batch.extra + assert vae.method_calls == [] and not vae.called + + +def test_input_stage_runs_on_a_synthetic_clip_for_warmup_requests(): + """Warmup requests carry no video (synthetic warmup) or a real one that must not be + decoded twice (request warmup): both run the real path on a small synthetic clip, + without audio, and without needing a prompt.""" + params = Kandinsky6SRSamplingParams(video_path="missing_on_purpose.mp4") + batch = Req(sampling_params=params) + batch.is_warmup = True + Kandinsky6SRInputStage().forward(batch, MagicMock()) + + video = batch.extra[SR_VIDEO_KEY] + height, width = WARMUP_CLIP_HW + # default scale 2.25 = x1.125 pre-upscale (aligned to 16) then tiling scale 2 + assert video.shape[0] == WARMUP_CLIP_FRAMES and video.dtype == torch.uint8 + assert video.shape[2:] == (288, 432) + # 288x432 is already a multiple of 16, so alignment padding is a no-op here, and the + # requested size the output stage will crop back to equals the padded (encoded) size. + assert batch.extra[SR_REQUESTED_HW_KEY] == (288, 432) + assert batch.extra["kandinsky6_sr_tiling_scale"] == 2 + assert (batch.fps, batch.num_frames) == (24, WARMUP_CLIP_FRAMES) + assert (batch.height, batch.width) == (288 * 2, 432 * 2) + assert batch.audio is None and height == 256 + + +def test_input_stage_pads_a_misaligned_source_to_the_spatial_factor(): + """A source whose pre-upscaled size is not already a multiple of 16 (every integer + scale, which runs no pixel resize at all) must still be padded before whole-video KVAE + encoding, and the *requested* (pre-pad) size recorded for the output stage to crop back + to -- the fix for "whole-video KVAE encoding fails at the fourth PixelUnshuffle" for + scales other than 2.25.""" + params = Kandinsky6SRSamplingParams( + video_path="missing_on_purpose.mp4", sr_resolution_scale=2 + ) + batch = Req(sampling_params=params) + batch.is_warmup = True + Kandinsky6SRInputStage().forward(batch, MagicMock()) + + # WARMUP_CLIP_HW is already 16-aligned, so exercise a genuinely misaligned size by + # checking the padding helper directly against the same video the stage produced, cut + # down to a misaligned size. + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + pad_to_spatial_factor, + ) + + misaligned = batch.extra[SR_VIDEO_KEY][:, :, :100, :70] + padded, requested_hw = pad_to_spatial_factor(misaligned, 16) + assert requested_hw == (100, 70) + assert padded.shape[-2] % 16 == 0 and padded.shape[-1] % 16 == 0 + assert torch.equal(padded[:, :, :100, :70], misaligned) + + +def test_input_stage_rejects_a_missing_video_file(): + batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="nope.mp4")) + with pytest.raises(ValueError, match="not found"): + Kandinsky6SRInputStage().forward(batch, MagicMock()) + + +def test_output_stage_stitches_crops_resizes_and_carries_the_source_audio(): + """The output stage blends tiles, restores the originally requested (pre-alignment-pad) + size, applies the delivery resize (fit keeps the aspect ratio and never upscales), + publishes the *final* size and fps on the request / result, and hands the source audio + through unchanged.""" + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + compute_tile_grid_even, + ) + + grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16) + plan = TilePlan( + tiling_scale=2, + frame_hw=(32, 48), + base_hw=(32, 48), + tile_hw=(16, 24), + pixel_grid=grid, + ) + tiles = [ + torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8) + for i in range(grid.total_tiles) + ] + audio = torch.linspace(-1, 1, 1000) + + params = Kandinsky6SRSamplingParams( + video_path="unused.mp4", + sr_target_resolution="48x32", + sr_target_resize_mode="fit", + ) + batch = Req(sampling_params=params) + batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan + batch.extra[SR_REQUESTED_HW_KEY] = (32, 48) # no alignment padding happened + batch.audio, batch.audio_sample_rate = audio, 44100 + batch.fps = 12 + + result = Kandinsky6SROutputStage().forward(batch, MagicMock()) + + assert result.output.shape == (1, 3, 9, 32, 48) # 64x96 fitted into 48x32 + assert (batch.height, batch.width) == (32, 48) + assert result.audio is audio and result.audio_sample_rate == 44100 + assert result.fps == 12 # the worker's effective fps reaches OutputBatch + plain = Kandinsky6SRSamplingParams(video_path="unused.mp4") + batch = Req(sampling_params=plain) + batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan + assert Kandinsky6SROutputStage().forward(batch, MagicMock()).output.shape == ( + 1, + 3, + 9, + 64, + 96, + ) + + +def test_output_stage_crops_the_alignment_padding_back_off(): + """A plan whose stitched frame is larger than what was actually requested (the input + stage padded for alignment) must be cropped down to the requested size before any + delivery resize -- not left at the padded size.""" + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + compute_tile_grid_even, + ) + + # Padded frame is 32x48 (aligned), but only 25x40 pixels were actually requested. + grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16) + plan = TilePlan( + tiling_scale=2, + frame_hw=(32, 48), + base_hw=(32, 48), + tile_hw=(16, 24), + pixel_grid=grid, + ) + tiles = [ + torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8) + for i in range(grid.total_tiles) + ] + batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="unused.mp4")) + batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan + batch.extra[SR_REQUESTED_HW_KEY] = (25, 40) + + result = Kandinsky6SROutputStage().forward(batch, MagicMock()) + + assert result.output.shape == (1, 3, 9, 50, 80) # (25, 40) * tiling_scale(2) + assert (batch.height, batch.width) == (50, 80) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py new file mode 100644 index 0000000000..6fe9f05341 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_tiling.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tile blending and initial latent precision.""" + +import pytest +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( + DitSpec, + SamplingSpec, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( + build_chunk_latent, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import ( + compute_tile_grid_even, + stitch_tiles_hanning, +) + + +@pytest.mark.parametrize( + "height,width,tile_hw", + [ + (512, 512, (512, 512)), + (512, 768, (512, 768)), + (768, 512, (768, 512)), + (768, 1536, (512, 768)), + ], +) +@pytest.mark.parametrize("value", [0.0, 1.0, 255.0]) +def test_hann_blending_preserves_constant_color_including_corners( + height, width, tile_hw, value +): + # tiny but nonzero Hann corner weights must not be clamped before normalization + grid = compute_tile_grid_even(height, width, tile_hw, 0.2, 16) + tiles = [torch.full((3, 1, *tile_hw), value) for _ in range(grid.total_tiles)] + result = stitch_tiles_hanning(tiles, grid, height, width, scale=1) + torch.testing.assert_close( + result, torch.full_like(result, value), rtol=1e-5, atol=1e-2 + ) + + +def test_uncovered_pixels_are_zero_not_nan(): + grid = compute_tile_grid_even(64, 64, (64, 64), 0.2, 16) + tile = torch.full((3, 1, 64, 64), 100.0) + result = stitch_tiles_hanning([tile], grid, 64, 128, scale=1) + assert result.shape == (3, 1, 64, 128) + torch.testing.assert_close(result[..., :64], tile, rtol=1e-5, atol=1e-2) + assert torch.equal(result[..., 64:], torch.zeros_like(result[..., 64:])) + + +@pytest.mark.parametrize("dit_dtype", [torch.bfloat16, torch.float32, None]) +def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype): + dit_spec = DitSpec( + instruct_type="noise", + visual_cond=False, + in_visual_dim=4, + use_motion_score=False, + patch_size=(1, 1, 1), + dtype=dit_dtype, + ) + spec = SamplingSpec( + tiling_scale=2, + tiles_batch_size=1, + seed=1, + num_steps=5, + tile_min_overlap=0.2, + visual_size=512, + scale_factor=(1.0, 1.0, 1.0), + scheduler_scale=5.0, + lq_noise_scale=0.7, + lq_noise_type="linear", + lq_channel_noise_scale=0.0, + cap_noise_timestep=False, + piflow=None, + ) + tile = torch.randn(4, 4, 4, 4, dtype=torch.float64) + result = build_chunk_latent( + [tile], + seed=1, + dit_spec=dit_spec, + spec=spec, + device=torch.device("cpu"), + ) + assert result.dtype == (dit_dtype or tile.dtype) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py new file mode 100644 index 0000000000..696518f141 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_video_io.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Streaming input / output helpers of Kandinsky 6 video SR (fps and frame budget). + +The reference decodes every frame and then applies ``resample_to_target_fps`` and +``clip_to_aligned_frames``; the port streams. These tests pin that both give the same +frames, and that the output tensor survives the framework's uint8 conversion. +""" + +import numpy as np +import pytest +import torch + +from sglang.multimodal_gen.configs.sample.kandinsky6_sr import ( + Kandinsky6SRSamplingParams, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import ( + Kandinsky6SRInputStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( + SR_TILING_SCALE_KEY, + SR_VIDEO_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import ( + FrameSelector, + decode_clip, + extract_source_audio, + to_output_video, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import ( + clip_to_aligned_frames, + resample_to_target_fps, +) + +FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000] +SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120] + + +def _stream_select(total: int, fps: float) -> tuple[list[int], int]: + """Run the streaming selector over frames ``0 .. total - 1`` (early stop included).""" + selector = FrameSelector(fps) + kept: list[int] = [] + seen = 0 + for index in range(total): + if selector.keep(index): + kept.append(index) + selector.mark_kept() + seen += 1 + if selector.enough(seen): + break + return kept[: selector.limit(seen, len(kept))], selector.effective_fps + + +def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None: + ids = torch.arange(total).view(total, 1, 1, 1) + resampled, effective_fps = resample_to_target_fps(ids, fps) + try: + clipped = clip_to_aligned_frames(resampled) + except ValueError: + return None + return clipped.flatten().tolist(), effective_fps + + +@pytest.mark.parametrize("fps", SOURCE_FPS) +def test_streaming_selection_equals_reference_resample_and_clip(fps): + """The stream may stop early once 121 frames are kept, but must return exactly the + frames (and the effective fps) of decode-all -> resample_to_target_fps -> + clip_to_aligned_frames, including sources too short for the 121-frame budget and + ``int(total / step)`` truncation of the last selectable frame.""" + for total in FRAME_COUNTS: + expected = _reference_select(total, fps) + kept, effective_fps = _stream_select(total, fps) + if expected is None: + # the reference raises "no readable frames"; nothing usable was selected + assert not kept, (total, fps) + continue + ids = torch.tensor(kept).view(-1, 1, 1, 1) + actual = clip_to_aligned_frames(ids).flatten().tolist() + assert (actual, effective_fps) == expected, (total, fps) + + +def test_effective_fps_is_an_int_by_the_reference_rule(): + """Downsampled sources play at exactly 24, in-tolerance / slow sources keep + ``round(source fps)`` (never a float: SamplingParams.fps must be a positive int).""" + assert FrameSelector(60.0).effective_fps == 24 + assert FrameSelector(25.0).effective_fps == 25 + assert FrameSelector(23.976).effective_fps == 24 + assert FrameSelector(12.0).effective_fps == 12 + assert all(isinstance(FrameSelector(fps).effective_fps, int) for fps in SOURCE_FPS) + + +# --------------------------------------------------------------------------- # +# Real files (PyAV) +# --------------------------------------------------------------------------- # +av = pytest.importorskip("av") + + +def write_test_video(path, *, frames, fps, size=(48, 64), audio_seconds=None): + """Tiny mp4 whose frame ``i`` is a flat colour ``i`` (so index errors are visible).""" + from fractions import Fraction + + height, width = size + with av.open(str(path), mode="w") as container: + video = container.add_stream("mpeg4", rate=fps) + video.width, video.height, video.pix_fmt = width, height, "yuv420p" + audio = None + if audio_seconds is not None: + audio = container.add_stream("aac", rate=22050) + audio.layout = "mono" + for index in range(frames): + gray = (index * 5) % 250 + array = np.full((height, width, 3), gray, dtype=np.uint8) + frame = av.VideoFrame.from_ndarray(array, format="rgb24") + frame.pts, frame.time_base = index, Fraction(1, fps) + for packet in video.encode(frame): + container.mux(packet) + if audio is not None: + t = np.arange(int(22050 * audio_seconds)) / 22050 + samples = (0.3 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)[None] + frame = av.AudioFrame.from_ndarray(samples, format="fltp", layout="mono") + frame.sample_rate, frame.pts = 22050, 0 + frame.time_base = Fraction(1, 22050) + for packet in audio.encode(frame): + container.mux(packet) + for packet in audio.encode(): + container.mux(packet) + for packet in video.encode(): + container.mux(packet) + + +def _decode_everything(path): + with av.open(str(path)) as container: + stream = container.streams.video[0] + fps = float(stream.average_rate) + frames = [ + torch.from_numpy(f.to_ndarray(format="rgb24")).permute(2, 0, 1) + for f in container.decode(stream) + ] + return torch.stack(frames), fps + + +@pytest.mark.parametrize("fps,frames", [(30, 40), (24, 26), (12, 20), (60, 80)]) +def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames): + """decode_clip == read_video_tchw_uint8 -> resample_to_target_fps -> + clip_to_aligned_frames: same pixels, same frame count (1 + 8k), same output fps.""" + path = tmp_path / "clip.mp4" + write_test_video(path, frames=frames, fps=fps) + full, source_fps = _decode_everything(path) + expected, expected_fps = resample_to_target_fps(full, source_fps) + expected = clip_to_aligned_frames(expected) + + clip = decode_clip(str(path)) + + assert clip.frames.dtype == torch.uint8 and clip.frames.shape[1] == 3 + assert clip.frames.shape[0] % 8 == 1 + assert torch.equal(clip.frames, expected) + assert clip.fps == expected_fps and isinstance(clip.fps, int) + + +def test_extract_source_audio_is_mono_float_trimmed_and_optional(tmp_path): + """Audio is decoded to mono float32 at 44.1 kHz, trimmed to the processed duration + (the reference muxes the whole track, leaving audio longer than the video), and a + file without an audio stream yields None instead of a silent track.""" + with_audio = tmp_path / "with_audio.mp4" + write_test_video(with_audio, frames=10, fps=10, audio_seconds=1.0) + full = extract_source_audio(str(with_audio)) + trimmed = extract_source_audio(str(with_audio), max_seconds=0.25) + assert full.dtype == np.float32 and full.ndim == 1 + assert abs(len(full) - 44100) < 2048 # aac priming / resampler delay + assert len(trimmed) == 11025 + assert np.abs(full).max() <= 1.0 and np.abs(full).max() > 0.05 + + silent = tmp_path / "silent.mp4" + write_test_video(silent, frames=10, fps=10) + assert extract_source_audio(str(silent)) is None + + +def test_output_video_round_trips_through_uint8_conversions(): + """Every uint8 value must survive the framework writer's ``(x * 255)`` truncation, + also when a consumer upcasts the fp16 tensor before multiplying (a plain ``k / 255`` + stored in fp16 fails that order for over half of the values).""" + values = torch.arange(256, dtype=torch.uint8) + video = values.view(1, 1, 8, 32).expand(3, 2, 8, 32).contiguous() + output = to_output_video(video) + assert output.dtype == torch.float16 and output.shape == (1, 3, 2, 8, 32) + + frames = _sample_to_uint8_frames(output[0]) # framework writer: list of [H, W, 3] + restored = torch.from_numpy(np.stack(frames)).permute(3, 0, 1, 2) + assert torch.equal(restored, video) + assert torch.equal((output.float() * 255).to(torch.uint8)[0], video) + + naive = (video.float() / 255.0).to(torch.float16) + assert not torch.equal((naive.float() * 255).to(torch.uint8), video) + + +@pytest.mark.parametrize( + "scale,size,expected_frame,tiling_scale", + [ + (2, (48, 64), (48, 64), 2), + (2.25, (64, 96), (64, 112), 2), + (4, (48, 64), (48, 64), 4), + ], +) +def test_input_stage_fills_the_request_from_a_video_file( + tmp_path, scale, size, expected_frame, tiling_scale +): + """A 30 fps source is thinned to 24 fps (an int), floored to 1 + 8k frames, and the + request's output geometry follows the clip: x1.125 pre-upscale aligned to 16 for the + 2.25 scale, the tile scale for the rest. The audio is mono float32 at 44.1 kHz and + trimmed to the processed duration.""" + path = tmp_path / "clip.mp4" + write_test_video(path, frames=40, fps=30, size=size, audio_seconds=1.5) + params = Kandinsky6SRSamplingParams(video_path=str(path), sr_resolution_scale=scale) + batch = Req(sampling_params=params) + + Kandinsky6SRInputStage().forward(batch, None) + + video = batch.extra[SR_VIDEO_KEY] + assert video.dtype == torch.uint8 and video.shape[1] == 3 + assert video.shape[0] == 25 and batch.num_frames == 25 # int(40 / 1.25) = 32 -> 25 + assert tuple(video.shape[2:]) == expected_frame + assert batch.extra[SR_TILING_SCALE_KEY] == tiling_scale + assert batch.fps == 24 and isinstance(batch.fps, int) + assert (batch.height, batch.width) == ( + expected_frame[0] * tiling_scale, + expected_frame[1] * tiling_scale, + ) + assert batch.audio.dtype == torch.float32 and batch.audio.ndim == 1 + assert batch.audio.shape[0] == round(25 / 24 * 44100) + assert batch.audio_sample_rate == 44100 + + +def test_input_stage_produces_no_audio_for_a_silent_source(tmp_path): + path = tmp_path / "silent.mp4" + write_test_video(path, frames=9, fps=24) + batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path=str(path))) + Kandinsky6SRInputStage().forward(batch, None) + assert batch.audio is None and batch.audio_sample_rate is None + assert batch.fps == 24 and batch.num_frames == 9 diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py new file mode 100644 index 0000000000..95d014a2c7 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_stages.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Image batching and audio/video alignment through real Kandinsky 6 stages.""" + +import math +from types import SimpleNamespace + +import PIL.Image +import pytest +import torch + +from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import ( + Kandinsky6TI2VAPipelineConfig, +) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + VideoGenerationsRequest, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import ( + _build_video_sampling_params, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import select_output_audio +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import ( + decoding, + image_encoding, + latent_preparation, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.decoding import ( + Kandinsky6AudioDecodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import ( + TAIL_COND_ACTIVE_EXTRA_KEY, + VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY, + Kandinsky6ImageEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.latent_preparation import ( + Kandinsky6LatentPreparationStage, +) + + +@pytest.fixture +def server_args(monkeypatch): + args = SimpleNamespace( + pipeline_config=Kandinsky6TI2VAPipelineConfig( + vae_precision="fp32", audio_vae_precision="fp32" + ), + component_precisions={}, + disable_autocast=True, + pipeline_class_name="Kandinsky6TI2VAPipeline", + backend="auto", + model_id=None, + model_path="kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers", + served_model_name="kandinsky6-ti2va-test", + output_path=None, + comfyui_mode=False, + num_gpus=1, + ) + # residency hooks capture the global args when each stage is constructed + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.server_args.server_args._global_server_args", + args, + ) + for module in (decoding, image_encoding, latent_preparation): + monkeypatch.setattr( + module, "get_local_torch_device", lambda: torch.device("cpu") + ) + return args + + +class ImageVAE: + def __init__(self, latent): + self.use_tiling = False + self.scaling_factor = 1.0 + self.shift_factor = None + self.latent = latent + + def to(self, *args, **kwargs): + return self + + def encode(self, image): + return SimpleNamespace(sample=lambda generator=None: self.latent) + + +@pytest.mark.parametrize("batch_size", [1, 2]) +def test_image_request_broadcasts_one_reference_to_each_output(server_args, batch_size): + request = VideoGenerationsRequest( + prompt="a cat playing piano", + input_reference="assets/girl.png", + num_outputs_per_prompt=batch_size, + ) + params = _build_video_sampling_params("k6-ti2va-multi-output", request) + assert params.num_outputs_per_prompt == batch_size + assert params.image_path == "assets/girl.png" + image_latent = torch.randn(1, 4, 1, 4, 6) + batch = Req( + sampling_params=params, + condition_image=PIL.Image.new("RGB", (96, 64)), + latents=torch.randn(batch_size, 3, 4, 6, 9), + height=64, + width=96, + generator=None, + ) + result = Kandinsky6ImageEncodingStage(ImageVAE(image_latent)).forward( + batch, server_args + ) + assert result.latents.shape == (batch_size, 4, 4, 6, 9) + assert result.image_latent.shape[0] == batch_size + for reference in result.image_latent: + torch.testing.assert_close(reference, result.image_latent[0], rtol=0, atol=0) + assert result.extra[TAIL_COND_ACTIVE_EXTRA_KEY] is True + assert result.extra[VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY].shape == (batch_size, 4) + + +class AudioVAE(torch.nn.Module): + scaling_factor = 1.0 + mean_value = 0.0 + + def __init__(self, waveform): + super().__init__() + self.weight = torch.nn.Parameter(torch.zeros(1)) + self.waveform = waveform + + def wrapped_decode(self, latents): + return self.waveform + + +def test_audio_decoding_preserves_each_output_track(server_args): + values = [-0.5, 0.0, 0.5] + waveform = torch.stack([torch.full((128,), value) for value in values]).unsqueeze(1) + batch = Req(sampling_params=SamplingParams(), audio_latents=torch.zeros(3, 5, 2)) + result = Kandinsky6AudioDecodingStage(AudioVAE(waveform)).forward( + batch, server_args + ) + assert result.audio.shape == (3, 128) + for index, value in enumerate(values): + torch.testing.assert_close( + select_output_audio(result.audio, index), + torch.full((128,), value), + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("num_frames,fps", [(1, 24), (61, 24), (121, 24), (121, 30)]) +def test_latent_preparation_aligns_audio_to_requested_video( + server_args, num_frames, fps +): + config = server_args.pipeline_config + batch = Req( + prompt="a cat playing piano", + height=16, + width=16, + num_frames=num_frames, + fps=fps, + ) + result = Kandinsky6LatentPreparationStage().forward(batch, server_args) + assert result.latents.shape[1] == (num_frames - 1) // 4 + 1 + expected = math.ceil( + num_frames / fps * config.audio_sample_rate / config.audio_downsample_factor + ) + assert result.audio_latents.shape[1] == expected diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py index 29dd9fa573..175a7930b6 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py @@ -356,19 +356,49 @@ def test_lora_exact_file_url_needs_no_weight_name(tmp_path): ] -def test_view_merge_unmerges_by_inverse_without_owned_clone(): +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("offloaded", [False, True]) +def test_view_merge_restores_exact_base_after_strength_changes(dtype, offloaded): torch.manual_seed(0) - base = torch.nn.Linear(4, 4, bias=False) + base = torch.nn.Linear(4, 4, bias=False, dtype=dtype) original = base.weight.detach().clone() layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2, snapshot_base=False) assert layer is not None assert layer._base_is_view + stored = base.weight.detach() + if offloaded: + base._packed_weight_cpu = stored + base.weight.data = torch.empty(1, dtype=dtype) A = torch.randn(2, 4) B = torch.randn(4, 2) - layer.set_lora_weights(A, B, strength=0.5, clear_existing=True, merge_weights=True) - assert layer.merged + layer.set_lora_weights(A, B, clear_existing=True, merge_weights=False) assert layer._base_is_view - torch.testing.assert_close(layer.base_layer.weight, original + 0.5 * (B @ A)) + for strength in (1.0, 0.0, 0.5, 1.0): + layer.set_lora_weights( + A, B, strength=strength, clear_existing=True, merge_weights=True + ) + assert layer.merged + assert not layer._base_is_view + torch.testing.assert_close(layer.cpu_weight, original, rtol=0, atol=0) + if strength == 0: + torch.testing.assert_close(stored, original, rtol=0, atol=0) + else: + assert not torch.equal(stored, original) layer.unmerge_lora_weights() - torch.testing.assert_close(layer.base_layer.weight, original) + torch.testing.assert_close(stored, original, rtol=0, atol=0) assert not layer.merged + + +def test_cached_merge_restores_untouched_base_view(): + base = torch.nn.Linear(4, 4, bias=False, dtype=torch.bfloat16) + original = base.weight.detach().clone() + layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2, snapshot_base=False) + layer.set_lora_weights(torch.ones(2, 4), torch.ones(4, 2), merge_weights=False) + layer.install_merged_weight(original + 2, base.weight.detach()) + assert layer._base_is_view + torch.testing.assert_close(layer.cpu_weight, original, rtol=0, atol=0) + layer.unmerge_lora_weights() + torch.testing.assert_close(base.weight, original, rtol=0, atol=0) + layer.merge_lora_weights(strength=0.5) + layer.unmerge_lora_weights() + torch.testing.assert_close(base.weight, original, rtol=0, atol=0) diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_ring_tail_pad.py b/python/sglang/multimodal_gen/test/unit/test_usp_ring_tail_pad.py index 7079677e95..d2dfbe99a4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_ring_tail_pad.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_ring_tail_pad.py @@ -7,6 +7,7 @@ from unittest.mock import patch import torch from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention +from sglang.multimodal_gen.runtime.layers.usp import _ring_merge_attention from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum _LAYER = "sglang.multimodal_gen.runtime.layers.attention.layer" @@ -26,8 +27,13 @@ class TestRingTailPadDispatch(unittest.TestCase): return obj def test_tail_pad_meta_reaches_the_ring_kernel(self): + for batch_size in (1, 2): + with self.subTest(batch_size=batch_size): + self._check_tail_pad(batch_size) + + def _check_tail_pad(self, batch_size): obj = self._attn() - q = torch.randn(1, 4, 2, 8) + q = torch.randn(batch_size, 4, 2, 8) meta = {"pad_start": 13, "pad_end": 16, "local_pad": 3} seen = {} @@ -46,7 +52,7 @@ class TestRingTailPadDispatch(unittest.TestCase): return_value=SimpleNamespace(attn_metadata=None), ), patch(f"{_LAYER}.get_sequence_parallel_world_size", return_value=4), - patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=2), + patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=4), patch( f"{_LAYER}._usp_input_all_to_all_qkv", side_effect=lambda q, k, v: (q, k, v), @@ -61,12 +67,25 @@ class TestRingTailPadDispatch(unittest.TestCase): self.assertEqual(out.shape, q.shape) self.assertEqual(seen["real"], 13) - self.assertEqual(seen["ws"], 2) + self.assertEqual(seen["ws"], 4) self.assertIs(seen["impl"], obj.attn_impl) - self.assertEqual(seen["shape"], (4, 2, 8)) + self.assertEqual(seen["shape"], (4, 2, 8) if batch_size == 1 else q.shape) # Last ring rank holds global rows [12, 16): row 13 onward is pad. - self.assertTrue(torch.all(out[0, 1:] == 0)) - self.assertTrue(torch.all(out[0, :1] == 1)) + self.assertTrue(torch.all(out[:, 1:] == 0)) + self.assertTrue(torch.all(out[:, :1] == 1)) + + def test_batched_merge_matches_full_attention(self): + torch.manual_seed(42) + q = torch.randn(2, 3, 5, 8) + k, v = torch.randn(2, 2, 3, 7, 8) + logits = q @ k.transpose(-2, -1) + expected = (logits.softmax(-1) @ v).transpose(1, 2) + out = lse = None + for start, end in ((0, 3), (3, 7)): + scores = logits[..., start:end] + partial = (scores.softmax(-1) @ v[..., start:end, :]).transpose(1, 2) + out, lse = _ring_merge_attention(out, lse, partial, scores.logsumexp(-1)) + torch.testing.assert_close(out, expected) def test_explicit_mask_under_ring_still_refuses(self): obj = self._attn() @@ -83,6 +102,26 @@ class TestRingTailPadDispatch(unittest.TestCase): with self.assertRaisesRegex(NotImplementedError, "ring"): obj.forward(q, q, q, attn_mask=mask) + def test_causal_tail_pad_refuses_local_only_causal_mask(self): + obj = self._attn() + obj.causal = True + q = torch.randn(2, 4, 2, 8) + with ( + patch( + f"{_LAYER}.get_forward_context", + return_value=SimpleNamespace(attn_metadata=None), + ), + patch(f"{_LAYER}.get_sequence_parallel_world_size", return_value=2), + patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=2), + ): + with self.assertRaisesRegex(NotImplementedError, "non-causal"): + obj.forward( + q, + q, + q, + attn_mask_meta={"pad_start": 7, "pad_end": 8, "local_pad": 1}, + ) + if __name__ == "__main__": unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_vae_tiled_decode_groups.py b/python/sglang/multimodal_gen/test/unit/test_vae_tiled_decode_groups.py new file mode 100644 index 0000000000..6b34a24435 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_vae_tiled_decode_groups.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tiled decoding must communicate within its SP group under TP x SP.""" + +from datetime import timedelta +from types import SimpleNamespace +from unittest.mock import patch + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig +from sglang.multimodal_gen.runtime.models.vaes import common + + +class _IdentityVAE(common.ParallelTiledVAE): + def _encode(self, x): + return x + + def _decode(self, z): + return z + + +def _decode_in_sp_group(rank, rendezvous): + dist.init_process_group( + "gloo", + init_method=rendezvous, + rank=rank, + world_size=4, + timeout=timedelta(seconds=45), + ) + try: + groups = [dist.new_group(ranks) for ranks in ([0, 2], [1, 3])] + group = groups[rank % 2] + coordinator = SimpleNamespace( + world_size=2, + rank_in_group=rank // 2, + device_group=group, + cpu_group=group, + ) + config = VAEConfig( + arch_config=VAEArchConfig( + temporal_compression_ratio=1, spatial_compression_ratio=1 + ), + tile_sample_min_height=2, + tile_sample_min_width=2, + tile_sample_stride_height=1, + tile_sample_stride_width=1, + tile_sample_min_num_frames=4, + tile_sample_stride_num_frames=3, + ) + vae = _IdentityVAE(config) + # Different content in each group detects cross-group contamination. + z = torch.arange(45, dtype=torch.float32).reshape(1, 1, 5, 3, 3) + z = z + 1000 * (rank % 2) + expected = vae.tiled_decode(z) + with patch.object(common, "get_sp_group", return_value=coordinator): + actual = vae.parallel_tiled_decode(z) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + finally: + dist.destroy_process_group() + + +def test_tiled_decode_uses_its_sp_group(tmp_path): + rendezvous = (tmp_path / "rendezvous").as_uri() + mp.spawn(_decode_in_sp_group, args=(rendezvous,), nprocs=4) diff --git a/test/registered/kernels/ops/diffusion/test_matrix_rope.py b/test/registered/kernels/ops/diffusion/test_matrix_rope.py new file mode 100644 index 0000000000..0ba3c23318 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_matrix_rope.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import apply_matrix_rope +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.version.hip is not None, + reason="NVIDIA CUDA required", +) + + +def reference(x, rope): + pairs = x.reshape(*x.shape[:-1], -1, 1, 2).float() + return (rope * pairs).sum(-1).reshape_as(x).to(x.dtype) + + +def inputs(shape, dtype, broadcast_batch=True): + torch.manual_seed(42) + batch, seq, _, dim = shape + x = torch.randn(shape, device="cuda", dtype=dtype) + rope = torch.randn( + 1 if broadcast_batch else batch, + seq + 3, + 1, + dim // 2, + 2, + 2, + device="cuda", + dtype=torch.float32, + )[:, 3:] + return x, rope + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("broadcast_batch", [False, True]) +@pytest.mark.parametrize( + "shape", [(1, 1, 1, 32), (2, 17, 3, 64), (2, 257, 4, 192), (1, 4096, 32, 128)] +) +def test_matrix_rope_bitexact(shape, dtype, broadcast_batch): + x, rope = inputs(shape, dtype, broadcast_batch) + original = x.clone() + actual = apply_matrix_rope(x, rope) + expected = reference(x, rope) + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + assert torch.equal(x, original) + assert actual.is_contiguous() + + +@pytest.mark.parametrize("dtype", [None, torch.float16, torch.bfloat16]) +def test_matrix_rope_compile_and_graph_replay(dtype): + x, rope = inputs((2, 257, 8, 128), torch.float32) + compiled = torch.compile(apply_matrix_rope, fullgraph=True) + assert torch.equal(compiled(x, rope, dtype), reference(x.to(dtype=dtype), rope)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + result = apply_matrix_rope(x, rope, dtype) + x.normal_() + rope.normal_() + graph.replay() + assert torch.equal(result, reference(x.to(dtype=dtype), rope)) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("input_dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("broadcast_batch", [False, True]) +@pytest.mark.parametrize("scale", [1e-4, 1.0, 100.0]) +def test_matrix_rope_cast_bitexact(dtype, input_dtype, broadcast_batch, scale): + x, rope = inputs((2, 257, 8, 128), input_dtype, broadcast_batch) + x *= scale + original = x.clone() + expected = reference(x.to(dtype), rope) + actual = apply_matrix_rope(x, rope, dtype) + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + assert torch.equal(x, original) + assert actual.dtype == dtype + assert actual.is_contiguous() + + +def test_matrix_rope_empty_input(): + x, rope = inputs((1, 0, 8, 128), torch.bfloat16) + assert apply_matrix_rope(x, rope).shape == x.shape + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/diffusion/test_residual_gate_fp32.py b/test/registered/kernels/ops/diffusion/test_residual_gate_fp32.py new file mode 100644 index 0000000000..1087928bfd --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_residual_gate_fp32.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import residual_gate_fp32 +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +requires_cuda = pytest.mark.skipif( + not torch.cuda.is_available() or torch.version.hip is not None, + reason="NVIDIA CUDA required", +) + + +def reference(x, update, gate): + return (x.float() + gate.float() * update.float()).to(x.dtype) + + +def inputs(shape, dtype, gate_dtype, broadcast_batch, device="cuda"): + torch.manual_seed(42) + x = torch.randn(shape, device=device, dtype=dtype) + update = torch.randn_like(x) + # modulation chunks have a nontrivial batch stride and storage offset + gate = torch.randn( + 1 if broadcast_batch else shape[0], + 1, + shape[-1] * 9, + device=device, + dtype=gate_dtype, + )[..., shape[-1] * 2 : shape[-1] * 3] + return x, update, gate + + +@requires_cuda +@torch.no_grad() +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("fp32_gate", [False, True]) +@pytest.mark.parametrize("broadcast_batch", [False, True]) +@pytest.mark.parametrize("shape", [(1, 1, 1), (2, 17, 67), (2, 1024, 4096)]) +def test_bitexact(shape, dtype, fp32_gate, broadcast_batch): + x, update, gate = inputs( + shape, dtype, torch.float32 if fp32_gate else dtype, broadcast_batch + ) + originals = [tensor.clone() for tensor in (x, update, gate)] + actual = residual_gate_fp32(x, update, gate) + expected = reference(x, update, gate) + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + for tensor, original in zip((x, update, gate), originals, strict=True): + assert torch.equal(tensor, original) + + +@requires_cuda +@torch.no_grad() +def test_fp32_rounding_not_fma_or_bf16_product(): + x = torch.tensor([[[-1.0, -0.435546875]]], device="cuda") + update = torch.tensor([[[1.0000001192092896, 0.48046875]]], device="cuda") + gate = torch.tensor([[[0.9999998807907104, 0.90625]]], device="cuda") + actual = residual_gate_fp32(x, update, gate) + assert torch.equal(actual, reference(x, update, gate)) + # the first product rounds to 1 before adding -1; FMA would retain -2**-46 + assert actual[0, 0, 0].item() == 0 + x, update, gate = (tensor.bfloat16() for tensor in (x, update, gate)) + expected = reference(x, update, gate) + assert not torch.equal(expected, x + gate * update) + assert torch.equal(residual_gate_fp32(x, update, gate), expected) + + +@requires_cuda +@torch.no_grad() +def test_compile_and_graph(): + x, update, gate = inputs((2, 257, 128), torch.bfloat16, torch.float32, False) + compiled = torch.compile(residual_gate_fp32, fullgraph=True) + assert torch.equal(compiled(x, update, gate), reference(x, update, gate)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = residual_gate_fp32(x, update, gate) + x.normal_() + update.normal_() + gate.normal_() + graph.replay() + assert torch.equal(actual, reference(x, update, gate)) + + +@pytest.mark.parametrize("device", ["cpu", pytest.param("cuda", marks=requires_cuda)]) +def test_fallback_and_empty(device): + x, update, gate = inputs((2, 17, 67), torch.float32, torch.float32, False, device) + x = x[:, ::2].requires_grad_() + update = update[:, ::2].requires_grad_() + gate.requires_grad_() + actual = residual_gate_fp32(x, update, gate) + expected = reference(x, update, gate) + assert torch.equal(actual, expected) + for actual_grad, expected_grad in zip( + torch.autograd.grad(actual.sum(), (x, update, gate)), + torch.autograd.grad(expected.sum(), (x, update, gate)), + strict=True, + ): + assert torch.equal(actual_grad, expected_grad) + with torch.no_grad(): + assert torch.equal(residual_gate_fp32(x, update, gate), expected) + assert residual_gate_fp32(x[:, :0], update[:, :0], gate).shape == (2, 0, 67) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))